PR: (Permanent draft) EKR's study code

docs/index.rst

@@ -15,6 +15,7 @@ Welcome to rope's documentation!
    library
    configuration
    contributing
+   theory
    release-process
    dev/issues
 

docs/theory.rst

@@ -0,0 +1,57 @@
+.. rst3: filename: docs/theory
+
+.. _`python's ast module`: https://docs.python.org/3/library/ast.html
+
+=====================
+How Rope infers types
+=====================
+
+This is the Theory of Operation for Rope's type inference,
+the most complex part of Rope.
+
+Only Rope's core devs need to understand this material.
+
+Some familiarity with `Python's ast module`_ is recommended, but not essential.
+
+.. contents:: Table of Contents
+
+Why Rope must infer types
+-------------------------
+
+.. To do.
+
+Overview of Rope's code base
+----------------------------
+
+- Startup code.
+- Type inference code.
+- Utility code.
+- Refactoring code.
+- Code completion.
+
+Overview of type inference
+--------------------------
+
+
+First principles:
+- Traversers do most of the work.
+- Local inference is easy; global inference is hard.
+
+Learning what to ignore is important.
+- @saveit is part of Rope's startup code. It has no direct part in type inference.
+
+What *not* to ignore:
+- @prevent_recursion prevents endless inference loops.
+
+What is an inference?
++++++++++++++++++++++
+
+Further study
+-------------
+
+Deep study of a few unit tests is recommended.
+
+`ObjectInferTest.test_simple_type_inferencing` is a good place to start.
+
+.. --- Insert traces here ---
+

rope/base/arguments.py

@@ -1,6 +1,19 @@
+from __future__ import annotations
+from typing import Any, Union, TYPE_CHECKING
 import rope.base.evaluate
 from rope.base import ast
 
+if TYPE_CHECKING:
+    from rope.base.pyobjects import PyFunction
+    from rope.base.pyobjectsdef import PyFunction as DefinedPyFunction
+    from rope.base.pyscopes import Scope
+
+    PyFunc = Union[PyFunction, DefinedPyFunction]
+else:
+    PyFunc = Any
+    Scope = Any
+Node = ast.AST
+
 
 class Arguments:
     """A class for evaluating parameters passed to a function
@@ -41,10 +54,15 @@ def _evaluate(self, ast_node):
         return rope.base.evaluate.eval_node(self.scope, ast_node)
 
 
-def create_arguments(primary, pyfunction, call_node, scope):
+# Call(expr func, expr* args, keyword* keywords)
+
+
+def create_arguments(
+    primary, pyfunction: PyFunc, call_node: ast.Call, scope: Scope
+) -> Arguments:
     """A factory for creating `Arguments`"""
     args = list(call_node.args)
-    args.extend(call_node.keywords)
+    args.extend(call_node.keywords)  # type:ignore
     called = call_node.func
     # XXX: Handle constructors
     if _is_method_call(primary, pyfunction) and isinstance(called, ast.Attribute):
@@ -94,7 +112,7 @@ def get_instance_pyname(self):
         return self.pyname
 
 
-def _is_method_call(primary, pyfunction):
+def _is_method_call(primary: Any, pyfunction: PyFunc) -> bool:
     if primary is None:
         return False
     pyobject = primary.get_object()

rope/base/ast.py

@@ -4,8 +4,12 @@
 
 from rope.base import fscommands
 
+from rope.base.utils import tracing_utils as g
+
+assert g
+
 try:
-    from ast import _const_node_type_names
+    from ast import _const_node_type_names  # type:ignore
 except ImportError:
     # backported from stdlib `ast`
     assert sys.version_info < (3, 8)
@@ -58,10 +62,15 @@ def call_for_nodes(node, callback):
 
 
 class RopeNodeVisitor(ast.NodeVisitor):
+    # This is the only visit method in Rope.
     def visit(self, node):
         """Modified from ast.NodeVisitor to match rope's existing Visitor implementation"""
         method = "_" + node.__class__.__name__
         visitor = getattr(self, method, self.generic_visit)
+        if 0:  # trace
+            module, name = visitor.__module__, visitor.__name__
+            if name != "generic_visit":
+                print(g.format("visit", module, name))
         return visitor(node)
 
 

rope/base/builtins.py

@@ -5,13 +5,19 @@
 import rope.base.evaluate
 from rope.base import arguments, ast, pynames, pyobjects, utils
 
+from rope.base.utils import tracing_utils as g
+
+assert g
+
 
 class BuiltinModule(pyobjects.AbstractModule):
     def __init__(self, name, pycore=None, initial={}):
         super().__init__()
         self.name = name
         self.pycore = pycore
         self.initial = initial
+        if 0:  # trace
+            print(g.format_ctor("builtins.BuiltinModule", __file__), name)
 
     parent = None
 

rope/base/change.py

@@ -6,7 +6,7 @@
 
 import rope.base.fscommands
 from rope.base import exceptions, taskhandle, utils
-from rope.base.fscommands import FileContent
+from rope.base.fscommands import FileContent  # type:ignore
 
 
 class Change:
@@ -334,7 +334,7 @@ def _get_fscommands(self, resource):
     def write_file(self, resource, contents: Union[str, FileContent]):
         data: FileContent
         if not isinstance(contents, bytes):
-            data = rope.base.fscommands.unicode_to_file_data(
+            data = rope.base.fscommands.unicode_to_file_data(  # type:ignore
                 contents,
                 newlines=resource.newlines,
             )

rope/base/evaluate.py

@@ -1,5 +1,6 @@
+from __future__ import annotations
 from operator import itemgetter
-from typing import Optional, Tuple
+from typing import Optional, Tuple, TYPE_CHECKING
 
 import rope.base.builtins
 import rope.base.pynames
@@ -14,6 +15,9 @@
     worder,
 )
 
+if TYPE_CHECKING:
+    from rope.base.pyscopes import Scope
+
 BadIdentifierError = exceptions.BadIdentifierError
 
 
@@ -158,7 +162,7 @@ def _find_module(self, module_name):
 
 
 class StatementEvaluator(ast.RopeNodeVisitor):
-    def __init__(self, scope):
+    def __init__(self, scope: Scope):
         self.scope = scope
         self.result = None
         self.old_result = None

rope/base/fscommands.py

@@ -1,3 +1,4 @@
+# type:ignore
 """Project file system commands.
 
 This modules implements file system operations used by rope.  Different

rope/base/libutils.py

@@ -81,7 +81,7 @@ def analyze_modules(project, task_handle=taskhandle.DEFAULT_TASK_HANDLE):
 
 
 def get_string_module(project, code, resource=None, force_errors=False):
-    """Returns a `PyObject` object for the given code
+    """Instantiates a `PyModule` object for the given code.
 
     If `force_errors` is `True`, `exceptions.ModuleSyntaxError` is
     raised if module has syntax errors.  This overrides
@@ -94,7 +94,7 @@ def get_string_module(project, code, resource=None, force_errors=False):
 
 
 def get_string_scope(project, code, resource=None):
-    """Returns a `Scope` object for the given code"""
+    """Instantiates a `Scope` object for the given code"""
     return get_string_module(project, code, resource).get_scope()
 
 

rope/base/nameanalyze.py

@@ -1,3 +1,4 @@
+from typing import List
 from rope.base import ast
 
 
@@ -19,8 +20,8 @@ def get_name_levels(node):
 
 
 class _NodeNameCollector(ast.RopeNodeVisitor):
-    def __init__(self, levels=None):
-        self.names = []
+    def __init__(self, levels: List[int] = None):
+        self.names: List[str] = []
         self.levels = levels
         self.index = 0
 

rope/base/oi/doa.py

@@ -1,3 +1,4 @@
+# type:ignore
 import base64
 import contextlib
 import hashlib

rope/base/oi/runmod.py

@@ -4,7 +4,7 @@ def __rope_start_everything():
     import sys
 
     try:
-        import cPickle as pickle
+        import cPickle as pickle  # type:ignore
     except ImportError:
         import pickle
     import base64

rope/base/oi/soa.py

@@ -1,8 +1,20 @@
+# type:ignore
+from __future__ import annotations
+from typing import Any, Union, TYPE_CHECKING
+
 import rope.base.ast
 import rope.base.oi.soi
 import rope.base.pynames
 from rope.base import arguments, evaluate, nameanalyze, pyobjects
 
+if TYPE_CHECKING:
+    from rope.base.pyobjects import PyFunction
+    from rope.base.pyobjectsdef import PyFunction as DefinedPyFunction
+
+    PyFunc = Union[PyFunction, DefinedPyFunction]
+else:
+    PyFunc = Any
+
 
 def analyze_module(pycore, pymodule, should_analyze, search_subscopes, followed_calls):
     """Analyze `pymodule` for static object inference
@@ -25,7 +37,7 @@ def _analyze_node(pycore, pydefined, should_analyze, search_subscopes, followed_
         return_true = lambda pydefined: True
         return_false = lambda pydefined: False
 
-        def _follow(pyfunction):
+        def _follow(pyfunction: PyFunc) -> None:
             _analyze_node(
                 pycore, pyfunction, return_true, return_false, new_followed_calls
             )
@@ -70,7 +82,7 @@ def _Call(self, node):
             return
         self._call(pyfunction, args)
 
-    def _args_with_self(self, primary, self_pyname, pyfunction, node):
+    def _args_with_self(self, primary, self_pyname, pyfunction: PyFunc, node: None):
         base_args = arguments.create_arguments(primary, pyfunction, node, self.scope)
         return arguments.MixedArguments(self_pyname, base_args, self.scope)
 

rope/base/oi/soi.py

@@ -4,15 +4,28 @@
 package.
 
 """
+from __future__ import annotations
+from typing import Any, Union, TYPE_CHECKING
 import rope.base.builtins  # Use full qualification for clarity.
 from rope.base import arguments, evaluate, pynames, pyobjects, utils
 from rope.base.oi.type_hinting.factory import get_type_hinting_factory
 
+if TYPE_CHECKING:
+    import ast
+    from rope.base.pyobjects import PyFunction
+    from rope.base.pyobjectsdef import PyFunction as DefinedPyFunction
+
+    Node = ast.AST
+    PyFunc = Union[PyFunction, DefinedPyFunction]
+else:
+    Node = Any
+    PyFunc = Any
+
 _ignore_inferred = utils.ignore_exception(pyobjects.IsBeingInferredError)
 
 
 @_ignore_inferred
-def infer_returned_object(pyfunction, args):
+def infer_returned_object(pyfunction: PyFunc, args):
     """Infer the `PyObject` this `PyFunction` returns after calling"""
     object_info = pyfunction.pycore.object_info
     result = object_info.get_exact_returned(pyfunction, args)
@@ -36,7 +49,7 @@ def infer_returned_object(pyfunction, args):
 
 
 @_ignore_inferred
-def infer_parameter_objects(pyfunction):
+def infer_parameter_objects(pyfunction: PyFunc):
     """Infer the `PyObject` of parameters of this `PyFunction`"""
     object_info = pyfunction.pycore.object_info
     result = object_info.get_parameter_objects(pyfunction)
@@ -85,7 +98,7 @@ def infer_assigned_object(pyname):
     return result
 
 
-def get_passed_objects(pyfunction, parameter_index):
+def get_passed_objects(pyfunction: PyFunc, parameter_index: int) -> Any:
     object_info = pyfunction.pycore.object_info
     result = object_info.get_passed_objects(pyfunction, parameter_index)
     if not result:

rope/base/oi/type_hinting/evaluate.py

@@ -1,3 +1,4 @@
+# type:ignore
 # Based on super lightweight Simple Top-Down Parser from http://effbot.org/zone/simple-top-down-parsing.htm
 # and https://bitbucket.org/emacsway/sqlbuilder/src/default/sqlbuilder/smartsql/contrib/evaluate.py
 import re

rope/base/oi/type_hinting/providers/numpydocstrings.py

@@ -1,3 +1,4 @@
+# type: ignore
 """
 Some code extracted (or based on code) from:
 https://github.com/davidhalter/jedi/blob/b489019f5bd5750051122b94cc767df47751ecb7/jedi/evaluate/docstrings.py

rope/base/oi/type_hinting/providers/pep0484_type_comments.py

@@ -1,6 +1,6 @@
 import re
 
-from rope.base.oi.type_hinting import utils
+# from rope.base.oi.type_hinting import utils
 from rope.base.oi.type_hinting.providers import interfaces
 
 

rope/base/oi/type_hinting/utils.py

@@ -1,5 +1,6 @@
+# type:ignore
 import logging
-from typing import Optional, Union
+from typing import Optional, Tuple
 
 import rope.base.utils as base_utils
 from rope.base import evaluate
@@ -72,8 +73,9 @@ def get_mro(pyclass):
     return class_list
 
 
-def resolve_type(type_name, pyobject):
-    # type: (str, Union[PyDefinedObject, PyObject]) -> Optional[PyDefinedObject, PyObject]
+def resolve_type(
+    type_name: str, pyobject: PyObject
+) -> Optional[Tuple[PyDefinedObject, PyObject]]:
     """
     Find proper type object from its name.
     """

rope/base/prefs.py

@@ -1,3 +1,4 @@
+# type: ignore
 """Rope preferences."""
 from dataclasses import asdict, dataclass
 from textwrap import dedent