Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Add machinery for marking places we subvert the type system (#5467)
We need this for compiling with mypyc, which inserts runtime
typechecks that cause problems when we subvert the type system. So
when compiling with mypyc, we turn those places into Any, while
keeping the types around for normal typechecks.

Since this causes the runtype types to be Any, this is best used
in places where efficient access to properties is not important.
For those cases some other technique should be used.
  • Loading branch information
msullivan committed Aug 15, 2018
1 parent 9093291 commit 5700456
Show file tree
Hide file tree
Showing 8 changed files with 80 additions and 37 deletions.
22 changes: 22 additions & 0 deletions mypy/bogus_type.py
@@ -0,0 +1,22 @@
"""A Bogus[T] type alias for marking when we subvert the type system
We need this for compiling with mypyc, which inserts runtime
typechecks that cause problems when we subvert the type system. So
when compiling with mypyc, we turn those places into Any, while
keeping the types around for normal typechecks.
Since this causes the runtime types to be Any, this is best used
in places where efficient access to properties is not important.
For those cases some other technique should be used.
"""

from mypy_extensions import FlexibleAlias
from typing import TypeVar, Any

T = TypeVar('T')

SUPPRESS_BOGUS_TYPES = False
if SUPPRESS_BOGUS_TYPES:
Bogus = FlexibleAlias[T, Any]
else:
Bogus = FlexibleAlias[T, T]
33 changes: 17 additions & 16 deletions mypy/nodes.py
Expand Up @@ -16,6 +16,8 @@
from mypy.visitor import NodeVisitor, StatementVisitor, ExpressionVisitor
from mypy.mypyc_hacks import SymbolTable

from mypy.bogus_type import Bogus


class Context:
"""Base type for objects that are valid as error message locations."""
Expand Down Expand Up @@ -177,8 +179,11 @@ class SymbolNode(Node):
@abstractmethod
def name(self) -> str: pass

# fullname can often be None even though the type system
# disagrees. We mark this with Bogus to let mypyc know not to
# worry about it.
@abstractmethod
def fullname(self) -> str: pass
def fullname(self) -> Bogus[str]: pass

@abstractmethod
def serialize(self) -> JsonDict: pass
Expand All @@ -195,10 +200,8 @@ def deserialize(cls, data: JsonDict) -> 'SymbolNode':
class MypyFile(SymbolNode):
"""The abstract syntax tree of a single source file."""

# Module name ('__main__' for initial file)
_name = None # type: str
# Fully qualified module name
_fullname = None # type: str
_fullname = None # type: Bogus[str]
# Path to the file (None if not known)
path = ''
# Top-level definitions and statements
Expand Down Expand Up @@ -238,9 +241,9 @@ def __init__(self,
self.ignored_lines = set()

def name(self) -> str:
return self._name
return '' if not self._fullname else self._fullname.split('.')[-1]

def fullname(self) -> str:
def fullname(self) -> Bogus[str]:
return self._fullname

def accept(self, visitor: NodeVisitor[T]) -> T:
Expand All @@ -252,7 +255,6 @@ def is_package_init_file(self) -> bool:

def serialize(self) -> JsonDict:
return {'.class': 'MypyFile',
'_name': self._name,
'_fullname': self._fullname,
'names': self.names.serialize(self._fullname),
'is_stub': self.is_stub,
Expand All @@ -264,7 +266,6 @@ def serialize(self) -> JsonDict:
def deserialize(cls, data: JsonDict) -> 'MypyFile':
assert data['.class'] == 'MypyFile', data
tree = MypyFile([], [])
tree._name = data['_name']
tree._fullname = data['_fullname']
tree.names = SymbolTable.deserialize(data['names'])
tree.is_stub = data['is_stub']
Expand Down Expand Up @@ -405,12 +406,12 @@ def __init__(self) -> None:
self.is_static = False
# Name with module prefix
# TODO: Type should be Optional[str]
self._fullname = cast(str, None)
self._fullname = cast(Bogus[str], None)

@abstractmethod
def name(self) -> str: pass

def fullname(self) -> str:
def fullname(self) -> Bogus[str]:
return self._fullname


Expand Down Expand Up @@ -660,7 +661,7 @@ def __init__(self, func: FuncDef, decorators: List[Expression],
def name(self) -> str:
return self.func.name()

def fullname(self) -> str:
def fullname(self) -> Bogus[str]:
return self.func.fullname()

@property
Expand Down Expand Up @@ -725,7 +726,7 @@ def __init__(self, name: str, type: 'Optional[mypy.types.Type]' = None) -> None:
super().__init__()
self._name = name # Name without module prefix
# TODO: Should be Optional[str]
self._fullname = cast(str, None) # Name with module prefix
self._fullname = cast(Bogus[str], None) # Name with module prefix
# TODO: Should be Optional[TypeInfo]
self.info = VAR_NO_INFO
self.type = type # type: Optional[mypy.types.Type] # Declared or inferred type, or None
Expand All @@ -748,7 +749,7 @@ def __init__(self, name: str, type: 'Optional[mypy.types.Type]' = None) -> None:
def name(self) -> str:
return self._name

def fullname(self) -> str:
def fullname(self) -> Bogus[str]:
return self._fullname

def accept(self, visitor: StatementVisitor[T]) -> T:
Expand Down Expand Up @@ -780,7 +781,7 @@ class ClassDef(Statement):
"""Class definition"""

name = None # type: str # Name of the class without module prefix
fullname = None # type: str # Fully qualified name of the class
fullname = None # type: Bogus[str] # Fully qualified name of the class
defs = None # type: Block
type_vars = None # type: List[mypy.types.TypeVarDef]
# Base class expressions (not semantically analyzed -- can be arbitrary expressions)
Expand Down Expand Up @@ -2053,7 +2054,7 @@ class is generic then it will be a type constructor of higher kind.
the appropriate number of arguments.
"""

_fullname = None # type: str # Fully qualified name
_fullname = None # type: Bogus[str] # Fully qualified name
# Fully qualified name for the module this type was defined in. This
# information is also in the fullname, but is harder to extract in the
# case of nested class definitions.
Expand Down Expand Up @@ -2193,7 +2194,7 @@ def name(self) -> str:
"""Short name."""
return self.defn.name

def fullname(self) -> str:
def fullname(self) -> Bogus[str]:
return self._fullname

def is_generic(self) -> bool:
Expand Down
1 change: 0 additions & 1 deletion mypy/treetransform.py
Expand Up @@ -61,7 +61,6 @@ def visit_mypy_file(self, node: MypyFile) -> MypyFile:
# NOTE: The 'names' and 'imports' instance variables will be empty!
new = MypyFile(self.statements(node.defs), [], node.is_bom,
ignored_lines=set(node.ignored_lines))
new._name = node._name
new._fullname = node._fullname
new.path = node.path
new.names = SymbolTable()
Expand Down
43 changes: 23 additions & 20 deletions mypy/types.py
Expand Up @@ -21,6 +21,7 @@
)
from mypy.sharedparse import argument_elide_name
from mypy.util import IdMapper
from mypy.bogus_type import Bogus

from mypy.mypyc_hacks import TypeOfAny

Expand Down Expand Up @@ -320,8 +321,9 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T:
return visitor.visit_any(self)

def copy_modified(self,
type_of_any: TypeOfAny = _dummy,
original_any: Optional['AnyType'] = _dummy,
# Mark with Bogus because _dummy is just an object (with type Any)
type_of_any: Bogus[TypeOfAny] = _dummy,
original_any: Bogus[Optional['AnyType']] = _dummy,
) -> 'AnyType':
if type_of_any is _dummy:
type_of_any = self.type_of_any
Expand Down Expand Up @@ -720,22 +722,22 @@ def __init__(self,
self.def_extras = {}

def copy_modified(self,
arg_types: List[Type] = _dummy,
arg_kinds: List[int] = _dummy,
arg_names: List[Optional[str]] = _dummy,
ret_type: Type = _dummy,
fallback: Instance = _dummy,
name: Optional[str] = _dummy,
definition: SymbolNode = _dummy,
variables: List[TypeVarDef] = _dummy,
line: int = _dummy,
column: int = _dummy,
is_ellipsis_args: bool = _dummy,
implicit: bool = _dummy,
special_sig: Optional[str] = _dummy,
from_type_type: bool = _dummy,
bound_args: List[Optional[Type]] = _dummy,
def_extras: Dict[str, Any] = _dummy) -> 'CallableType':
arg_types: Bogus[List[Type]] = _dummy,
arg_kinds: Bogus[List[int]] = _dummy,
arg_names: Bogus[List[Optional[str]]] = _dummy,
ret_type: Bogus[Type] = _dummy,
fallback: Bogus[Instance] = _dummy,
name: Bogus[Optional[str]] = _dummy,
definition: Bogus[SymbolNode] = _dummy,
variables: Bogus[List[TypeVarDef]] = _dummy,
line: Bogus[int] = _dummy,
column: Bogus[int] = _dummy,
is_ellipsis_args: Bogus[bool] = _dummy,
implicit: Bogus[bool] = _dummy,
special_sig: Bogus[Optional[str]] = _dummy,
from_type_type: Bogus[bool] = _dummy,
bound_args: Bogus[List[Optional[Type]]] = _dummy,
def_extras: Bogus[Dict[str, Any]] = _dummy) -> 'CallableType':
return CallableType(
arg_types=arg_types if arg_types is not _dummy else self.arg_types,
arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds,
Expand Down Expand Up @@ -1434,8 +1436,9 @@ class TypeType(Type):
# a generic class instance, a union, Any, a type variable...
item = None # type: Type

def __init__(self, item: Union[Instance, AnyType, TypeVarType, TupleType, NoneTyp,
CallableType], *, line: int = -1, column: int = -1) -> None:
def __init__(self, item: Bogus[Union[Instance, AnyType, TypeVarType, TupleType, NoneTyp,
CallableType]], *,
line: int = -1, column: int = -1) -> None:
"""To ensure Type[Union[A, B]] is always represented as Union[Type[A], Type[B]], item of
type UnionType must be handled through make_normalized static method.
"""
Expand Down
15 changes: 15 additions & 0 deletions mypy_bootstrap.ini
@@ -0,0 +1,15 @@
[mypy]
disallow_untyped_defs = True
disallow_subclassing_any = True
warn_no_return = True
strict_optional = True
no_implicit_optional = True
disallow_any_generics = True
disallow_any_unimported = True
warn_redundant_casts = True
warn_unused_configs = True
always_true = SUPPRESS_BOGUS_TYPES

# needs py2 compatibility
[mypy-mypy.test.testextensions]
disallow_untyped_defs = False
1 change: 1 addition & 0 deletions mypy_self_check.ini
Expand Up @@ -8,6 +8,7 @@ disallow_any_generics = True
disallow_any_unimported = True
warn_redundant_casts = True
warn_unused_configs = True
always_false = SUPPRESS_BOGUS_TYPES

# needs py2 compatibility
[mypy-mypy.test.testextensions]
Expand Down
1 change: 1 addition & 0 deletions setup.py
Expand Up @@ -104,6 +104,7 @@ def run(self):
classifiers=classifiers,
cmdclass={'build_py': CustomPythonBuild},
install_requires = ['typed-ast >= 1.1.0, < 1.2.0',
'mypy_extensions >= 0.4.0, < 0.5.0',
],
extras_require = {
':python_version < "3.5"': 'typing >= 3.5.3',
Expand Down
1 change: 1 addition & 0 deletions test-requirements.txt
Expand Up @@ -3,6 +3,7 @@ flake8
flake8-bugbear; python_version >= '3.5'
flake8-pyi; python_version >= '3.6'
lxml==4.2.4
mypy_extensions==0.4.0
psutil==5.4.0
pytest>=3.4
pytest-xdist>=1.22
Expand Down

0 comments on commit 5700456

Please sign in to comment.