From 08349542b673174f31f30627acc602b0c026edb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Wed, 28 Nov 2018 22:03:01 +0100 Subject: [PATCH 01/19] Parse docstrings to custom objects. --- mypy/stubgenc.py | 97 +++++++++--------- mypy/stubutil.py | 62 ++++++++++- mypy/test/teststubgen.py | 215 +++++++++++++++++++++++++++++++++++---- 3 files changed, 302 insertions(+), 72 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 751103bca469..61abcf460ed0 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -12,7 +12,8 @@ from mypy.stubutil import ( is_c_module, write_header, infer_sig_from_docstring, - infer_prop_type_from_docstring + infer_prop_type_from_docstring, ArgList, TypedArgSig, + infer_arg_sig_from_docstring, TypedFunctionSig ) @@ -123,51 +124,40 @@ def generate_c_function_stub(module: ModuleType, ) -> None: ret_type = 'None' if name == '__init__' and class_name else 'Any' - if self_var: - self_arg = '%s, ' % self_var - else: - self_arg = '' if (name in ('__new__', '__init__') and name not in sigs and class_name and class_name in class_sigs): - sig = class_sigs[class_name] + inferred = TypedFunctionSig( + name=name, + args=infer_arg_sig_from_docstring(class_sigs[class_name]), + ret_type=ret_type + ) else: docstr = getattr(obj, '__doc__', None) inferred = infer_sig_from_docstring(docstr, name) - if inferred: - sig, ret_type = inferred - else: + if not inferred: if class_name and name not in sigs: - sig = infer_method_sig(name) + inferred = TypedFunctionSig(name, args=infer_method_sig(name), ret_type=ret_type) else: - sig = sigs.get(name, '(*args, **kwargs)') - # strip away parenthesis - sig = sig[1:-1] - if sig: - if self_var: - # remove annotation on self from signature if present - groups = sig.split(',', 1) - if groups[0] == self_var or groups[0].startswith(self_var + ':'): - self_arg = '' - sig = '{},{}'.format(self_var, groups[1]) if len(groups) > 1 else self_var - else: - self_arg = self_arg.replace(', ', '') - - if sig: - sig_types = [] - # convert signature in form of "self: TestClass, arg0: str" to - # list [[self, TestClass], [arg0, str]] - for arg in sig.split(','): - arg_type = arg.split(':', 1) - if len(arg_type) == 1: - # there is no type provided in docstring - sig_types.append(arg_type[0].strip()) - else: - arg_type_name = strip_or_import(arg_type[1].strip(), module, imports) - sig_types.append('%s: %s' % (arg_type[0].strip(), arg_type_name)) - sig = ", ".join(sig_types) + inferred = TypedFunctionSig( + name=name, + args=infer_arg_sig_from_docstring(sigs.get(name, '(*args, **kwargs)')), + ret_type=ret_type + ) + + sig = [] + for arg in inferred.args: + if arg.name == self_var or not arg.type: + # no type + sig.append(arg.name) + else: + # type info + sig.append('{}: {}'.format(arg.name, strip_or_import(arg.type, module, imports))) - ret_type = strip_or_import(ret_type, module, imports) - output.append('def %s(%s%s) -> %s: ...' % (name, self_arg, sig, ret_type)) + output.append('def {function}({args}) -> {ret}: ...'.format( + function=name, + args=", ".join(sig), + ret=strip_or_import(inferred.ret_type, module, imports) + )) def strip_or_import(typ: str, module: ModuleType, imports: List[str]) -> str: @@ -307,29 +297,38 @@ def is_skipped_attribute(attr: str) -> bool: '__weakref__') # For pickling -def infer_method_sig(name: str) -> str: +def infer_method_sig(name: str) -> ArgList: if name.startswith('__') and name.endswith('__'): name = name[2:-2] if name in ('hash', 'iter', 'next', 'sizeof', 'copy', 'deepcopy', 'reduce', 'getinitargs', 'int', 'float', 'trunc', 'complex', 'bool'): - return '()' + return [] if name == 'getitem': - return '(index)' + return [TypedArgSig(name='index')] if name == 'setitem': - return '(index, object)' + return [ + TypedArgSig(name='index'), + TypedArgSig(name='object') + ] if name in ('delattr', 'getattr'): - return '(name)' + return [TypedArgSig(name='name')] if name == 'setattr': - return '(name, value)' + return [ + TypedArgSig(name='name'), + TypedArgSig(name='value') + ] if name == 'getstate': - return '()' + return [] if name == 'setstate': - return '(state)' + return [TypedArgSig(name='state')] if name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul', 'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv', 'divmod', 'rdivmod', 'pow', 'rpow'): - return '(other)' + return [TypedArgSig(name='other')] if name in ('neg', 'pos'): - return '()' - return '(*args, **kwargs)' + return [] + return [ + TypedArgSig(name='*args'), + TypedArgSig(name='**kwargs') + ] diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 9f6b4b8a03b0..24d9344534bf 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -2,7 +2,7 @@ import sys import os -from typing import Optional, Tuple, Sequence, MutableSequence, List, MutableMapping, IO +from typing import Optional, Tuple, Sequence, MutableSequence, List, MutableMapping, IO, NamedTuple from types import ModuleType @@ -10,6 +10,21 @@ Sig = Tuple[str, str] +class TypedArgSig(NamedTuple): + name: str + type: Optional[str] = None + default: Optional[str] = None + + +ArgList = List[TypedArgSig] + + +class TypedFunctionSig(NamedTuple): + name: str + args: ArgList + ret_type: str + + def parse_signature(sig: str) -> Optional[Tuple[str, List[str], List[str]]]: @@ -106,7 +121,7 @@ def write_header(file: IO[str], module_name: Optional[str] = None, '# NOTE: This dynamically typed stub was automatically generated by stubgen.\n\n') -def infer_sig_from_docstring(docstr: str, name: str) -> Optional[Tuple[str, str]]: +def infer_sig_from_docstring(docstr: str, name: str) -> Optional[TypedFunctionSig]: if not docstr: return None docstr = docstr.lstrip() @@ -125,15 +140,54 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[Tuple[str, str] m = re.match(sig_match + ' -> ([a-zA-Z].*)$', docstr, re.MULTILINE) if m: # strip potential white spaces at the right of return type - return m.group(1), m.group(2).rstrip() + return TypedFunctionSig( + name=name, + args=infer_arg_sig_from_docstring(m.group(1)), + ret_type=m.group(2).rstrip() + ) # try to not match return type m = re.match(sig_match, docstr) if m: - return m.group(1), 'Any' + return TypedFunctionSig( + name=name, + args=infer_arg_sig_from_docstring(m.group(1)), + ret_type='Any' + ) return None +def infer_arg_sig_from_docstring(docstr: str) -> ArgList: + """ + convert signature in form of "(self: TestClass, arg0: str='ada')" to ArgList + + :param docstr: + :return: ArgList with infered argument names and its types + """ + ret = [] # type: ArgList + arguments = [] + right = docstr[1:-1] + accumulator = "" + while right: + left, sep, right = right.partition(',') + if right.count('[') == right.count(']'): + arguments.append(accumulator + left) + accumulator = "" + else: + accumulator += left + sep + + for arg in arguments: + arg_name_type, _, default_value = arg.partition('=') + arg_name, _, arg_type = arg_name_type.partition(':') + + ret.append(TypedArgSig( + name=arg_name.strip(), + type=None if arg_type == '' else arg_type.strip(), + default=None if default_value == '' else default_value.strip() + )) + return ret + + def infer_prop_type_from_docstring(docstr: str) -> Optional[str]: if not docstr: return None diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 79b7877cb972..bdf3ac79b5ce 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -17,7 +17,7 @@ from mypy.stubgenc import generate_c_type_stub, infer_method_sig, generate_c_function_stub from mypy.stubutil import ( parse_signature, parse_all_signatures, build_signature, find_unique_signatures, - infer_sig_from_docstring, infer_prop_type_from_docstring + infer_sig_from_docstring, infer_prop_type_from_docstring, TypedFunctionSig, TypedArgSig ) @@ -103,24 +103,115 @@ def test_find_unique_signatures(self) -> None: ('func3', '(arg, arg2)')]) def test_infer_sig_from_docstring(self) -> None: - assert_equal(infer_sig_from_docstring('\nfunc(x) - y', 'func'), ('(x)', 'Any')) - assert_equal(infer_sig_from_docstring('\nfunc(x, Y_a=None)', 'func'), - ('(x, Y_a=None)', 'Any')) + assert_equal( + infer_sig_from_docstring('\nfunc(x) - y', 'func'), + TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x')], + ret_type='Any' + ) + ) + + assert_equal( + infer_sig_from_docstring('\nfunc(x, Y_a=None)', 'func'), + TypedFunctionSig( + name='func', + args=[ + TypedArgSig(name='x'), + TypedArgSig(name='Y_a', default='None') + ], + ret_type='Any' + ) + ) + assert_equal( + infer_sig_from_docstring('\nfunc(x, Y_a=3)', 'func'), + TypedFunctionSig( + name='func', + args=[ + TypedArgSig(name='x'), + TypedArgSig(name='Y_a', default='3') + ], + ret_type='Any' + ) + ) + + assert_equal( + infer_sig_from_docstring('\nfunc(x, Y_a=[1, 2, 3])', 'func'), + TypedFunctionSig( + name='func', + args=[ + TypedArgSig(name='x'), + TypedArgSig(name='Y_a', default='[1, 2, 3]') + ], + ret_type='Any' + ) + ) + assert_equal(infer_sig_from_docstring('\nafunc(x) - y', 'func'), None) assert_equal(infer_sig_from_docstring('\nfunc(x, y', 'func'), None) assert_equal(infer_sig_from_docstring('\nfunc(x=z(y))', 'func'), None) assert_equal(infer_sig_from_docstring('\nfunc x', 'func'), None) # try to infer signature from type annotation - assert_equal(infer_sig_from_docstring('\nfunc(x: int)', 'func'), ('(x: int)', 'Any')) - assert_equal(infer_sig_from_docstring('\nfunc(x: int=3)', 'func'), ('(x: int=3)', 'Any')) - assert_equal(infer_sig_from_docstring('\nfunc(x: int=3) -> int', 'func'), - ('(x: int=3)', 'int')) - assert_equal(infer_sig_from_docstring('\nfunc(x: int=3) -> int \n', 'func'), - ('(x: int=3)', 'int')) - assert_equal(infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), - ('(x: Tuple[int, str])', 'str')) - assert_equal(infer_sig_from_docstring('\nfunc(x: foo.bar)', 'func'), - ('(x: foo.bar)', 'Any')) + assert_equal( + infer_sig_from_docstring('\nfunc(x: int)', 'func'), + TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='int')], + ret_type='Any' + ) + ) + assert_equal( + infer_sig_from_docstring('\nfunc(x: int=3)', 'func'), + TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='int', default='3')], + ret_type='Any' + ) + ) + assert_equal( + infer_sig_from_docstring('\nfunc(x: int=3) -> int', 'func'), + TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='int', default='3')], + ret_type='int' + ) + ) + assert_equal( + infer_sig_from_docstring('\nfunc(x: int=3) -> int \n', 'func'), + TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='int', default='3')], + ret_type='int' + ) + ) + assert_equal( + infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), + TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='Tuple[int, str]')], + ret_type='str' + ) + ) + assert_equal( + infer_sig_from_docstring('\nfunc(x: Tuple[int, Tuple[str, int], str], y: int) -> str', + 'func'), + TypedFunctionSig( + name='func', + args=[ + TypedArgSig(name='x', type='Tuple[int, Tuple[str, int], str]'), + TypedArgSig(name='y', type='int'), + ], + ret_type='str' + ) + ) + assert_equal( + infer_sig_from_docstring('\nfunc(x: foo.bar)', 'func'), + TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='foo.bar')], + ret_type='Any' + ) + ) def infer_prop_type_from_docstring(self) -> None: assert_equal(infer_prop_type_from_docstring('str: A string.'), 'str') @@ -219,22 +310,25 @@ def add_file(path: str, result: List[str]) -> None: class StubgencSuite(Suite): def test_infer_hash_sig(self) -> None: - assert_equal(infer_method_sig('__hash__'), '()') + assert_equal(infer_method_sig('__hash__'), []) def test_infer_getitem_sig(self) -> None: - assert_equal(infer_method_sig('__getitem__'), '(index)') + assert_equal(infer_method_sig('__getitem__'), [TypedArgSig(name='index')]) def test_infer_setitem_sig(self) -> None: - assert_equal(infer_method_sig('__setitem__'), '(index, object)') + assert_equal(infer_method_sig('__setitem__'), [ + TypedArgSig(name='index'), + TypedArgSig(name='object') + ]) def test_infer_binary_op_sig(self) -> None: for op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul'): - assert_equal(infer_method_sig('__%s__' % op), '(other)') + assert_equal(infer_method_sig('__%s__' % op), [TypedArgSig(name='other')]) def test_infer_unary_op_sig(self) -> None: for op in ('neg', 'pos'): - assert_equal(infer_method_sig('__%s__' % op), '()') + assert_equal(infer_method_sig('__%s__' % op), []) def test_generate_c_type_stub_no_crash_for_object(self) -> None: output = [] # type: List[str] @@ -310,3 +404,86 @@ def test(self, arg0: str) -> None: 'def test(self, arg0: int) -> Any: ...' ]) assert_equal(imports, []) + + def test_generate_c_function_other_module_arg(self) -> None: + """ + Test that if argument references type from other module, module will be imported + """ + # provide different type in python spec than in docstring to make sure, that docstring + # information is used + def test(arg0: str) -> None: + """ + test(arg0: argparse.Action) + """ + pass + output = [] # type: List[str] + imports = [] # type: List[str] + mod = ModuleType(self.__module__, '') + generate_c_function_stub(mod, 'test', test, output, imports) + assert_equal(output, [ + 'def test(arg0: argparse.Action) -> Any: ...' + ]) + assert_equal(imports, [ + 'import argparse' + ]) + + def test_generate_c_function_same_module_arg(self) -> None: + """ + Test that if argument references type from same module but using full path, no module will + be imported, and type specification will be striped to local reference + """ + # provide different type in python spec than in docstring to make sure, that docstring + # information is used + def test(arg0: str) -> None: + """ + test(arg0: argparse.Action) + """ + pass + output = [] # type: List[str] + imports = [] # type: List[str] + mod = ModuleType('argparse', '') + generate_c_function_stub(mod, 'test', test, output, imports) + assert_equal(output, [ + 'def test(arg0: Action) -> Any: ...' + ]) + assert_equal(imports, []) + + def test_generate_c_function_other_module_ret(self) -> None: + """ + Test that if return type references type from other module, module will be imported + """ + def test(arg0: str) -> None: + """ + test(arg0: str) -> argparse.Action + """ + pass + output = [] # type: List[str] + imports = [] # type: List[str] + mod = ModuleType(self.__module__, '') + generate_c_function_stub(mod, 'test', test, output, imports) + assert_equal(output, [ + 'def test(arg0: str) -> argparse.Action: ...' + ]) + assert_equal(imports, [ + 'import argparse' + ]) + + def test_generate_c_function_same_module_ret(self) -> None: + """ + Test that if return type references type from same module but using full path, no module + will be imported, and type specification will be striped to local reference + """ + def test(arg0: str) -> None: + """ + test(arg0: str) -> argparse.Action + """ + pass + output = [] # type: List[str] + imports = [] # type: List[str] + mod = ModuleType('argparse', '') + generate_c_function_stub(mod, 'test', test, output, imports) + assert_equal(output, [ + 'def test(arg0: str) -> Action: ...' + ]) + assert_equal(imports, []) + From 7035a9731a10332e965eb1b58c6c419323f88954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 29 Nov 2018 22:22:36 +0100 Subject: [PATCH 02/19] Support for overloaded function generated by pybind11 --- mypy/stubgenc.py | 64 ++++++++++++----------- mypy/stubutil.py | 83 +++++++++++++++++++----------- mypy/test/teststubgen.py | 106 +++++++++++++++++++++++++-------------- 3 files changed, 156 insertions(+), 97 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 61abcf460ed0..38506437ca21 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -126,38 +126,44 @@ def generate_c_function_stub(module: ModuleType, if (name in ('__new__', '__init__') and name not in sigs and class_name and class_name in class_sigs): - inferred = TypedFunctionSig( + inferred = [TypedFunctionSig( name=name, args=infer_arg_sig_from_docstring(class_sigs[class_name]), ret_type=ret_type - ) + )] # type: Optional[List[TypedFunctionSig]] else: docstr = getattr(obj, '__doc__', None) inferred = infer_sig_from_docstring(docstr, name) if not inferred: if class_name and name not in sigs: - inferred = TypedFunctionSig(name, args=infer_method_sig(name), ret_type=ret_type) + inferred = [TypedFunctionSig(name, args=infer_method_sig(name), ret_type=ret_type)] else: - inferred = TypedFunctionSig( + inferred = [TypedFunctionSig( name=name, args=infer_arg_sig_from_docstring(sigs.get(name, '(*args, **kwargs)')), ret_type=ret_type - ) - - sig = [] - for arg in inferred.args: - if arg.name == self_var or not arg.type: - # no type - sig.append(arg.name) - else: - # type info - sig.append('{}: {}'.format(arg.name, strip_or_import(arg.type, module, imports))) + )] + + is_overloaded = len(inferred) > 1 + if is_overloaded: + imports.append('from typing import overload') + for signature in inferred: + sig = [] + for arg in signature.args: + if arg.name == self_var or not arg.type: + # no type + sig.append(arg.name) + else: + # type info + sig.append('{}: {}'.format(arg.name, strip_or_import(arg.type, module, imports))) - output.append('def {function}({args}) -> {ret}: ...'.format( - function=name, - args=", ".join(sig), - ret=strip_or_import(inferred.ret_type, module, imports) - )) + if is_overloaded: + output.append('@overload') + output.append('def {function}({args}) -> {ret}: ...'.format( + function=name, + args=", ".join(sig), + ret=strip_or_import(signature.ret_type, module, imports) + )) def strip_or_import(typ: str, module: ModuleType, imports: List[str]) -> str: @@ -304,31 +310,31 @@ def infer_method_sig(name: str) -> ArgList: 'int', 'float', 'trunc', 'complex', 'bool'): return [] if name == 'getitem': - return [TypedArgSig(name='index')] + return [TypedArgSig(name='index', type=None, default=None)] if name == 'setitem': return [ - TypedArgSig(name='index'), - TypedArgSig(name='object') + TypedArgSig(name='index', type=None, default=None), + TypedArgSig(name='object', type=None, default=None) ] if name in ('delattr', 'getattr'): - return [TypedArgSig(name='name')] + return [TypedArgSig(name='name', type=None, default=None)] if name == 'setattr': return [ - TypedArgSig(name='name'), - TypedArgSig(name='value') + TypedArgSig(name='name', type=None, default=None), + TypedArgSig(name='value', type=None, default=None) ] if name == 'getstate': return [] if name == 'setstate': - return [TypedArgSig(name='state')] + return [TypedArgSig(name='state', type=None, default=None)] if name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul', 'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv', 'divmod', 'rdivmod', 'pow', 'rpow'): - return [TypedArgSig(name='other')] + return [TypedArgSig(name='other', type=None, default=None)] if name in ('neg', 'pos'): return [] return [ - TypedArgSig(name='*args'), - TypedArgSig(name='**kwargs') + TypedArgSig(name='*args', type=None, default=None), + TypedArgSig(name='**kwargs', type=None, default=None) ] diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 24d9344534bf..f9b7a244bed2 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -9,20 +9,19 @@ # Type Alias for Signatures Sig = Tuple[str, str] - -class TypedArgSig(NamedTuple): - name: str - type: Optional[str] = None - default: Optional[str] = None - +TypedArgSig = NamedTuple('TypedArgSig', [ + ('name', str), + ('type', Optional[str]), + ('default', Optional[str]) +]) ArgList = List[TypedArgSig] - -class TypedFunctionSig(NamedTuple): - name: str - args: ArgList - ret_type: str +TypedFunctionSig = NamedTuple('TypedFunctionSig', [ + ('name', str), + ('args', ArgList), + ('ret_type', str) +]) def parse_signature(sig: str) -> Optional[Tuple[str, @@ -121,10 +120,11 @@ def write_header(file: IO[str], module_name: Optional[str] = None, '# NOTE: This dynamically typed stub was automatically generated by stubgen.\n\n') -def infer_sig_from_docstring(docstr: str, name: str) -> Optional[TypedFunctionSig]: +def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunctionSig]]: if not docstr: return None docstr = docstr.lstrip() + is_overloaded = any(('Overloaded function.' == x.strip()) for x in docstr.split('\n')) # look for function signature, which is any string of the format # () -> # or perhaps without the return type @@ -136,24 +136,47 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[TypedFunctionSi # to capture return type, sig_str = r'\([a-zA-Z0-9_=:, \[\]\.]*\)' sig_match = r'%s(%s)' % (name, sig_str) - # first, try to capture return type; we just match until end of line - m = re.match(sig_match + ' -> ([a-zA-Z].*)$', docstr, re.MULTILINE) - if m: - # strip potential white spaces at the right of return type - return TypedFunctionSig( - name=name, - args=infer_arg_sig_from_docstring(m.group(1)), - ret_type=m.group(2).rstrip() - ) - - # try to not match return type - m = re.match(sig_match, docstr) - if m: - return TypedFunctionSig( - name=name, - args=infer_arg_sig_from_docstring(m.group(1)), - ret_type='Any' - ) + sig_match_ret = sig_match + ' -> ([a-zA-Z].*)$' + + if is_overloaded: + def find_sig_ret() -> List[Tuple[str, str]]: + return re.findall(sig_match_ret, docstr, re.MULTILINE) + + def find_sig() -> List[str]: + return re.findall(sig_match, docstr, re.MULTILINE) + else: + def find_sig_ret() -> List[Tuple[str, str]]: + m = re.match(sig_match_ret, docstr, re.MULTILINE) + if m: + return [(m.group(1), m.group(2))] + return [] + + def find_sig() -> List[str]: + m = re.match(sig_match, docstr) + if m: + return [m.group(1)] + return [] + + sig_ret_match = find_sig_ret() + if sig_ret_match: + ret = [] + for match in sig_ret_match: + ret.append(TypedFunctionSig( + name=name, + args=infer_arg_sig_from_docstring(match[0]), + ret_type=match[1].rstrip() + )) + return ret + sig_match = find_sig() + if sig_match: + ret = [] + for match in sig_match: + ret.append(TypedFunctionSig( + name=name, + args=infer_arg_sig_from_docstring(match), + ret_type='Any' + )) + return ret return None diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index bdf3ac79b5ce..28c6f0567ed0 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -105,46 +105,46 @@ def test_find_unique_signatures(self) -> None: def test_infer_sig_from_docstring(self) -> None: assert_equal( infer_sig_from_docstring('\nfunc(x) - y', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x')], + args=[TypedArgSig(name='x', type=None, default=None)], ret_type='Any' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=None)', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x'), - TypedArgSig(name='Y_a', default='None') + TypedArgSig(name='x', type=None, default=None), + TypedArgSig(name='Y_a', type=None, default='None') ], ret_type='Any' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=3)', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x'), - TypedArgSig(name='Y_a', default='3') + TypedArgSig(name='x', type=None, default=None), + TypedArgSig(name='Y_a', type=None, default='3') ], ret_type='Any' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=[1, 2, 3])', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x'), - TypedArgSig(name='Y_a', default='[1, 2, 3]') + TypedArgSig(name='x', type=None, default=None), + TypedArgSig(name='Y_a', type=None, default='[1, 2, 3]') ], ret_type='Any' - ) + )] ) assert_equal(infer_sig_from_docstring('\nafunc(x) - y', 'func'), None) @@ -154,63 +154,63 @@ def test_infer_sig_from_docstring(self) -> None: # try to infer signature from type annotation assert_equal( infer_sig_from_docstring('\nfunc(x: int)', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='int')], + args=[TypedArgSig(name='x', type='int', default=None)], ret_type='Any' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3)', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', args=[TypedArgSig(name='x', type='int', default='3')], ret_type='Any' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3) -> int', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', args=[TypedArgSig(name='x', type='int', default='3')], ret_type='int' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3) -> int \n', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', args=[TypedArgSig(name='x', type='int', default='3')], ret_type='int' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='Tuple[int, str]')], + args=[TypedArgSig(name='x', type='Tuple[int, str]', default=None)], ret_type='str' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, Tuple[str, int], str], y: int) -> str', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x', type='Tuple[int, Tuple[str, int], str]'), - TypedArgSig(name='y', type='int'), + TypedArgSig(name='x', type='Tuple[int, Tuple[str, int], str]', default=None), + TypedArgSig(name='y', type='int', default=None), ], ret_type='str' - ) + )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: foo.bar)', 'func'), - TypedFunctionSig( + [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='foo.bar')], + args=[TypedArgSig(name='x', type='foo.bar', default=None)], ret_type='Any' - ) + )] ) def infer_prop_type_from_docstring(self) -> None: @@ -313,18 +313,22 @@ def test_infer_hash_sig(self) -> None: assert_equal(infer_method_sig('__hash__'), []) def test_infer_getitem_sig(self) -> None: - assert_equal(infer_method_sig('__getitem__'), [TypedArgSig(name='index')]) + assert_equal(infer_method_sig('__getitem__'), [TypedArgSig( + name='index', type=None, default=None + )]) def test_infer_setitem_sig(self) -> None: assert_equal(infer_method_sig('__setitem__'), [ - TypedArgSig(name='index'), - TypedArgSig(name='object') + TypedArgSig(name='index', type=None, default=None), + TypedArgSig(name='object', type=None, default=None) ]) def test_infer_binary_op_sig(self) -> None: for op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul'): - assert_equal(infer_method_sig('__%s__' % op), [TypedArgSig(name='other')]) + assert_equal(infer_method_sig('__%s__' % op), [TypedArgSig( + name='other', type=None, default=None + )]) def test_infer_unary_op_sig(self) -> None: for op in ('neg', 'pos'): @@ -487,3 +491,29 @@ def test(arg0: str) -> None: ]) assert_equal(imports, []) + def test_generate_c_type_with_overload_pybind11(self) -> None: + class TestClass: + def __init__(self, arg0: str) -> None: + """ + __init__(*args, **kwargs) + Overloaded function. + + 1. __init__(self: TestClass, arg0: str) -> None + + 2. __init__(self: TestClass, arg0: str, arg1: str) -> None + """ + pass + output = [] # type: List[str] + imports = [] # type: List[str] + mod = ModuleType(TestClass.__module__, '') + generate_c_function_stub(mod, '__init__', TestClass.__init__, output, imports, + self_var='self', class_name='TestClass') + assert_equal(output, [ + '@overload', + 'def __init__(self, arg0: str) -> None: ...', + '@overload', + 'def __init__(self, arg0: str, arg1: str) -> None: ...', + ]) + assert_equal(set(imports), { + 'from typing import overload' + }) From cf4de081b4b1c4fc8d3411c7d58c10582a7326f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 29 Nov 2018 23:39:22 +0100 Subject: [PATCH 03/19] Fix mypy self-check --- mypy/stubgenc.py | 37 +++++++++++++++++++------------------ mypy/stubutil.py | 16 ++++++++-------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 38506437ca21..fcbacd5d79db 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -144,26 +144,27 @@ def generate_c_function_stub(module: ModuleType, ret_type=ret_type )] - is_overloaded = len(inferred) > 1 + is_overloaded = len(inferred) > 1 if inferred else False if is_overloaded: imports.append('from typing import overload') - for signature in inferred: - sig = [] - for arg in signature.args: - if arg.name == self_var or not arg.type: - # no type - sig.append(arg.name) - else: - # type info - sig.append('{}: {}'.format(arg.name, strip_or_import(arg.type, module, imports))) - - if is_overloaded: - output.append('@overload') - output.append('def {function}({args}) -> {ret}: ...'.format( - function=name, - args=", ".join(sig), - ret=strip_or_import(signature.ret_type, module, imports) - )) + if inferred: + for signature in inferred: + sig = [] + for arg in signature.args: + if arg.name == self_var or not arg.type: + # no type + sig.append(arg.name) + else: + # type info + sig.append('{}: {}'.format(arg.name, strip_or_import(arg.type, module, imports))) + + if is_overloaded: + output.append('@overload') + output.append('def {function}({args}) -> {ret}: ...'.format( + function=name, + args=", ".join(sig), + ret=strip_or_import(signature.ret_type, module, imports) + )) def strip_or_import(typ: str, module: ModuleType, imports: List[str]) -> str: diff --git a/mypy/stubutil.py b/mypy/stubutil.py index f9b7a244bed2..a3a72f506e9c 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -157,20 +157,20 @@ def find_sig() -> List[str]: return [m.group(1)] return [] - sig_ret_match = find_sig_ret() - if sig_ret_match: + sig_match_ret_res = find_sig_ret() + if sig_match_ret_res: ret = [] - for match in sig_ret_match: + for match_ret in sig_match_ret_res: ret.append(TypedFunctionSig( name=name, - args=infer_arg_sig_from_docstring(match[0]), - ret_type=match[1].rstrip() + args=infer_arg_sig_from_docstring(match_ret[0]), + ret_type=match_ret[1].rstrip() )) return ret - sig_match = find_sig() - if sig_match: + sig_match_res = find_sig() + if sig_match_res: ret = [] - for match in sig_match: + for match in sig_match_res: ret.append(TypedFunctionSig( name=name, args=infer_arg_sig_from_docstring(match), From 38872dc3e63c1e88fa97112a9db591905f977089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Wed, 19 Dec 2018 21:32:23 +0100 Subject: [PATCH 04/19] Use tokenize to parse function declarations in docstr --- mypy/stubgenc.py | 4 +- mypy/stubutil.py | 161 ++++++++++++++++++++++++++------------- mypy/test/teststubgen.py | 23 ++++-- 3 files changed, 125 insertions(+), 63 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index fcbacd5d79db..f9d3d05ea807 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -156,7 +156,9 @@ def generate_c_function_stub(module: ModuleType, sig.append(arg.name) else: # type info - sig.append('{}: {}'.format(arg.name, strip_or_import(arg.type, module, imports))) + sig.append('{}: {}'.format(arg.name, strip_or_import(arg.type, + module, + imports))) if is_overloaded: output.append('@overload') diff --git a/mypy/stubutil.py b/mypy/stubutil.py index a3a72f506e9c..992540c71087 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -1,6 +1,9 @@ +import enum +import io import re import sys import os +import tokenize from typing import Optional, Tuple, Sequence, MutableSequence, List, MutableMapping, IO, NamedTuple from types import ModuleType @@ -120,64 +123,112 @@ def write_header(file: IO[str], module_name: Optional[str] = None, '# NOTE: This dynamically typed stub was automatically generated by stubgen.\n\n') +class State(enum.Enum): + INIT = 1 + FUNCTION_NAME = 2 + ARGUMENT_LIST = 3 + ARGUMENT_TYPE = 4 + ARGUMENT_DEFAULT = 5 + RETURN_VALUE = 6 + OPEN_BRACKET = 7 + + def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunctionSig]]: if not docstr: return None - docstr = docstr.lstrip() - is_overloaded = any(('Overloaded function.' == x.strip()) for x in docstr.split('\n')) - # look for function signature, which is any string of the format - # () -> - # or perhaps without the return type - - # in the signature, we allow the following characters: - # colon/equal: to match default values, like "a: int=1" - # comma/space/brackets: for type hints like "a: Tuple[int, float]" - # dot: for classes annotating using full path, like "a: foo.bar.baz" - # to capture return type, - sig_str = r'\([a-zA-Z0-9_=:, \[\]\.]*\)' - sig_match = r'%s(%s)' % (name, sig_str) - sig_match_ret = sig_match + ' -> ([a-zA-Z].*)$' - - if is_overloaded: - def find_sig_ret() -> List[Tuple[str, str]]: - return re.findall(sig_match_ret, docstr, re.MULTILINE) - - def find_sig() -> List[str]: - return re.findall(sig_match, docstr, re.MULTILINE) - else: - def find_sig_ret() -> List[Tuple[str, str]]: - m = re.match(sig_match_ret, docstr, re.MULTILINE) - if m: - return [(m.group(1), m.group(2))] - return [] - - def find_sig() -> List[str]: - m = re.match(sig_match, docstr) - if m: - return [m.group(1)] - return [] - - sig_match_ret_res = find_sig_ret() - if sig_match_ret_res: - ret = [] - for match_ret in sig_match_ret_res: - ret.append(TypedFunctionSig( - name=name, - args=infer_arg_sig_from_docstring(match_ret[0]), - ret_type=match_ret[1].rstrip() - )) - return ret - sig_match_res = find_sig() - if sig_match_res: - ret = [] - for match in sig_match_res: - ret.append(TypedFunctionSig( - name=name, - args=infer_arg_sig_from_docstring(match), - ret_type='Any' - )) - return ret - return None + + state = [State.INIT, ] + accumulator = "" + arg_type = None + arg_name = "" + arg_default = None + ret_type = "Any" + found = False + args = [] # type: List[TypedArgSig] + signatures = [] # type: List[TypedFunctionSig] + try: + for token in tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline): + if token.type == tokenize.NAME and token.string == name and state[-1] == State.INIT: + state.append(State.FUNCTION_NAME) + + elif token.type == tokenize.OP and token.string == '(' and state[-1] == \ + State.FUNCTION_NAME: + state.pop() + accumulator = "" + found = True + state.append(State.ARGUMENT_LIST) + + elif state[-1] == State.FUNCTION_NAME: + # reset state, function name not followed by '(' + state.pop() + + elif token.type == tokenize.OP and token.string in ('[', '(', '{'): + accumulator += token.string + state.append(State.OPEN_BRACKET) + + elif token.type == tokenize.OP and token.string in (']', ')', '}') and \ + state[-1] == State.OPEN_BRACKET: + accumulator += token.string + state.pop() + + elif token.type == tokenize.OP and token.string == ':' and \ + state[-1] == State.ARGUMENT_LIST: + arg_name = accumulator + accumulator = "" + state.append(State.ARGUMENT_TYPE) + + elif token.type == tokenize.OP and token.string == '=' and state[-1] in ( + State.ARGUMENT_LIST, State.ARGUMENT_TYPE): + if state[-1] == State.ARGUMENT_TYPE: + arg_type = accumulator + state.pop() + else: + arg_name = accumulator + accumulator = "" + state.append(State.ARGUMENT_DEFAULT) + + elif token.type == tokenize.OP and token.string in (',', ')') and state[-1] in ( + State.ARGUMENT_LIST, State.ARGUMENT_DEFAULT, State.ARGUMENT_TYPE): + if state[-1] == State.ARGUMENT_DEFAULT: + arg_default = accumulator + state.pop() + elif state[-1] == State.ARGUMENT_TYPE: + arg_type = accumulator + state.pop() + elif state[-1] == State.ARGUMENT_LIST: + arg_name = accumulator + + if token.string == ')': + state.pop() + args.append(TypedArgSig(name=arg_name, type=arg_type, default=arg_default)) + arg_name = "" + arg_type = None + arg_default = None + accumulator = "" + + elif token.type == tokenize.OP and token.string == '->': + accumulator = "" + state.append(State.RETURN_VALUE) + + elif token.type == tokenize.NEWLINE and state[-1] in (State.INIT, State.RETURN_VALUE): + if state[-1] == State.RETURN_VALUE: + ret_type = accumulator + accumulator = "" + state.pop() + + if found: + signatures.append(TypedFunctionSig(name=name, args=args, ret_type=ret_type)) + found = False + args = [] + ret_type = 'Any' + # leave state as INIT + else: + accumulator += token.string + + return signatures + except tokenize.TokenError: + # return as much as collected + return signatures def infer_arg_sig_from_docstring(docstr: str) -> ArgList: diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 28c6f0567ed0..779ff7832e00 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -141,16 +141,23 @@ def test_infer_sig_from_docstring(self) -> None: name='func', args=[ TypedArgSig(name='x', type=None, default=None), - TypedArgSig(name='Y_a', type=None, default='[1, 2, 3]') + TypedArgSig(name='Y_a', type=None, default='[1,2,3]') ], ret_type='Any' )] ) - assert_equal(infer_sig_from_docstring('\nafunc(x) - y', 'func'), None) - assert_equal(infer_sig_from_docstring('\nfunc(x, y', 'func'), None) - assert_equal(infer_sig_from_docstring('\nfunc(x=z(y))', 'func'), None) - assert_equal(infer_sig_from_docstring('\nfunc x', 'func'), None) + assert_equal(infer_sig_from_docstring('\nafunc(x) - y', 'func'), []) + assert_equal(infer_sig_from_docstring('\nfunc(x, y', 'func'), []) + assert_equal( + infer_sig_from_docstring('\nfunc(x=z(y))', 'func'), + [TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type=None, default='z(y)')], + ret_type='Any' + )] + ) + assert_equal(infer_sig_from_docstring('\nfunc x', 'func'), []) # try to infer signature from type annotation assert_equal( infer_sig_from_docstring('\nfunc(x: int)', 'func'), @@ -188,7 +195,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='Tuple[int, str]', default=None)], + args=[TypedArgSig(name='x', type='Tuple[int,str]', default=None)], ret_type='str' )] ) @@ -198,7 +205,7 @@ def test_infer_sig_from_docstring(self) -> None: [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x', type='Tuple[int, Tuple[str, int], str]', default=None), + TypedArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=None), TypedArgSig(name='y', type='int', default=None), ], ret_type='str' @@ -509,6 +516,8 @@ def __init__(self, arg0: str) -> None: generate_c_function_stub(mod, '__init__', TestClass.__init__, output, imports, self_var='self', class_name='TestClass') assert_equal(output, [ + '@overload', + 'def __init__(*args, **kwargs) -> Any: ...', '@overload', 'def __init__(self, arg0: str) -> None: ...', '@overload', From 860de835c1b4a27282c685258bee6d16834695d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 20 Dec 2018 00:32:26 +0100 Subject: [PATCH 05/19] Use ENDMARKER for python 3.4 --- mypy/stubutil.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 992540c71087..08f6029d7d2b 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -210,7 +210,9 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct accumulator = "" state.append(State.RETURN_VALUE) - elif token.type == tokenize.NEWLINE and state[-1] in (State.INIT, State.RETURN_VALUE): + # ENDMAKER is necessary for python 3.4 and 3.5 + elif token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and state[-1] in ( + State.INIT, State.RETURN_VALUE): if state[-1] == State.RETURN_VALUE: ret_type = accumulator accumulator = "" From ae08bd3e477f960455a02adf12d085b2249b6251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 20 Dec 2018 09:25:04 +0100 Subject: [PATCH 06/19] Always check for state --- mypy/stubutil.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 08f6029d7d2b..9a9aab38976c 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -162,7 +162,8 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct # reset state, function name not followed by '(' state.pop() - elif token.type == tokenize.OP and token.string in ('[', '(', '{'): + elif token.type == tokenize.OP and token.string in ('[', '(', '{') and \ + state[-1] != State.INIT: accumulator += token.string state.append(State.OPEN_BRACKET) @@ -206,7 +207,7 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct arg_default = None accumulator = "" - elif token.type == tokenize.OP and token.string == '->': + elif token.type == tokenize.OP and token.string == '->' and state[-1] == State.INIT: accumulator = "" state.append(State.RETURN_VALUE) From d1aee100f421a3508455a08db50f00f65e817493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Sat, 26 Jan 2019 12:19:52 +0100 Subject: [PATCH 07/19] Review fixes --- mypy/stubgenc.py | 17 +++++++---------- mypy/stubutil.py | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index f9d3d05ea807..7c6a45f0f176 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -126,11 +126,9 @@ def generate_c_function_stub(module: ModuleType, if (name in ('__new__', '__init__') and name not in sigs and class_name and class_name in class_sigs): - inferred = [TypedFunctionSig( - name=name, - args=infer_arg_sig_from_docstring(class_sigs[class_name]), - ret_type=ret_type - )] # type: Optional[List[TypedFunctionSig]] + inferred = [TypedFunctionSig(name=name, + args=infer_arg_sig_from_docstring(class_sigs[class_name]), + ret_type=ret_type)] # type: Optional[List[TypedFunctionSig]] else: docstr = getattr(obj, '__doc__', None) inferred = infer_sig_from_docstring(docstr, name) @@ -138,11 +136,10 @@ def generate_c_function_stub(module: ModuleType, if class_name and name not in sigs: inferred = [TypedFunctionSig(name, args=infer_method_sig(name), ret_type=ret_type)] else: - inferred = [TypedFunctionSig( - name=name, - args=infer_arg_sig_from_docstring(sigs.get(name, '(*args, **kwargs)')), - ret_type=ret_type - )] + inferred = [TypedFunctionSig(name=name, + args=infer_arg_sig_from_docstring( + sigs.get(name, '(*args, **kwargs)')), + ret_type=ret_type)] is_overloaded = len(inferred) > 1 if inferred else False if is_overloaded: diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 9a9aab38976c..1cd898378286 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -137,7 +137,7 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct if not docstr: return None - state = [State.INIT, ] + state = [State.INIT] accumulator = "" arg_type = None arg_name = "" @@ -151,8 +151,8 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct if token.type == tokenize.NAME and token.string == name and state[-1] == State.INIT: state.append(State.FUNCTION_NAME) - elif token.type == tokenize.OP and token.string == '(' and state[-1] == \ - State.FUNCTION_NAME: + elif (token.type == tokenize.OP and token.string == '(' and + state[-1] == State.FUNCTION_NAME): state.pop() accumulator = "" found = True @@ -162,24 +162,24 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct # reset state, function name not followed by '(' state.pop() - elif token.type == tokenize.OP and token.string in ('[', '(', '{') and \ - state[-1] != State.INIT: + elif (token.type == tokenize.OP and token.string in ('[', '(', '{') and + state[-1] != State.INIT): accumulator += token.string state.append(State.OPEN_BRACKET) - elif token.type == tokenize.OP and token.string in (']', ')', '}') and \ - state[-1] == State.OPEN_BRACKET: + elif (token.type == tokenize.OP and token.string in (']', ')', '}') and + state[-1] == State.OPEN_BRACKET): accumulator += token.string state.pop() - elif token.type == tokenize.OP and token.string == ':' and \ - state[-1] == State.ARGUMENT_LIST: + elif (token.type == tokenize.OP and token.string == ':' and + state[-1] == State.ARGUMENT_LIST): arg_name = accumulator accumulator = "" state.append(State.ARGUMENT_TYPE) - elif token.type == tokenize.OP and token.string == '=' and state[-1] in ( - State.ARGUMENT_LIST, State.ARGUMENT_TYPE): + elif (token.type == tokenize.OP and token.string == '=' and + state[-1] in (State.ARGUMENT_LIST, State.ARGUMENT_TYPE)): if state[-1] == State.ARGUMENT_TYPE: arg_type = accumulator state.pop() @@ -188,8 +188,8 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct accumulator = "" state.append(State.ARGUMENT_DEFAULT) - elif token.type == tokenize.OP and token.string in (',', ')') and state[-1] in ( - State.ARGUMENT_LIST, State.ARGUMENT_DEFAULT, State.ARGUMENT_TYPE): + elif (token.type == tokenize.OP and token.string in (',', ')') and + state[-1] in (State.ARGUMENT_LIST, State.ARGUMENT_DEFAULT, State.ARGUMENT_TYPE)): if state[-1] == State.ARGUMENT_DEFAULT: arg_default = accumulator state.pop() @@ -212,8 +212,8 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct state.append(State.RETURN_VALUE) # ENDMAKER is necessary for python 3.4 and 3.5 - elif token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and state[-1] in ( - State.INIT, State.RETURN_VALUE): + elif (token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and + state[-1] in (State.INIT, State.RETURN_VALUE)): if state[-1] == State.RETURN_VALUE: ret_type = accumulator accumulator = "" From 13048b57e3919520234572318074466d4c906329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Sat, 26 Jan 2019 12:35:35 +0100 Subject: [PATCH 08/19] Use infer_sig_from_docstring in infer_arg_sig_from_docstring, add tests --- mypy/stubutil.py | 27 ++++----------------- mypy/test/teststubgen.py | 52 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 24 deletions(-) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 1cd898378286..06e1e550d6b9 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -241,28 +241,11 @@ def infer_arg_sig_from_docstring(docstr: str) -> ArgList: :param docstr: :return: ArgList with infered argument names and its types """ - ret = [] # type: ArgList - arguments = [] - right = docstr[1:-1] - accumulator = "" - while right: - left, sep, right = right.partition(',') - if right.count('[') == right.count(']'): - arguments.append(accumulator + left) - accumulator = "" - else: - accumulator += left + sep - - for arg in arguments: - arg_name_type, _, default_value = arg.partition('=') - arg_name, _, arg_type = arg_name_type.partition(':') - - ret.append(TypedArgSig( - name=arg_name.strip(), - type=None if arg_type == '' else arg_type.strip(), - default=None if default_value == '' else default_value.strip() - )) - return ret + ret = infer_sig_from_docstring("stub" + docstr, "stub") + if ret: + return ret[0].args + + return [] def infer_prop_type_from_docstring(docstr: str) -> Optional[str]: diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 779ff7832e00..cbd54f68606e 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -17,7 +17,8 @@ from mypy.stubgenc import generate_c_type_stub, infer_method_sig, generate_c_function_stub from mypy.stubutil import ( parse_signature, parse_all_signatures, build_signature, find_unique_signatures, - infer_sig_from_docstring, infer_prop_type_from_docstring, TypedFunctionSig, TypedArgSig + infer_sig_from_docstring, infer_prop_type_from_docstring, TypedFunctionSig, TypedArgSig, + infer_arg_sig_from_docstring ) @@ -220,7 +221,54 @@ def test_infer_sig_from_docstring(self) -> None: )] ) - def infer_prop_type_from_docstring(self) -> None: + assert_equal( + infer_sig_from_docstring('\nfunc(x: list=[1,2,[3,4]])', 'func'), + [TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='list', default='[1,2,[3,4]]')], + ret_type='Any' + )] + ) + + assert_equal( + infer_sig_from_docstring('\nfunc(x: str="nasty[")', 'func'), + [TypedFunctionSig( + name='func', + args=[TypedArgSig(name='x', type='str', default='"nasty["')], + ret_type='Any' + )] + ) + + assert_equal( + infer_sig_from_docstring('\nfunc[(x: foo.bar, invalid]', 'func'), + [] + ) + + def test_infer_arg_sig_from_docstring(self) -> None: + assert_equal( + infer_arg_sig_from_docstring("(*args, **kwargs)"), + [ + TypedArgSig(name='*args', type=None, default=None), + TypedArgSig(name='**kwargs', type=None, default=None), + ] + ) + + assert_equal( + infer_arg_sig_from_docstring( + "(x: Tuple[int, Tuple[str, int], str]=(1, ('a', 2), 'y'), y: int=4)" + ), + [ + TypedArgSig( + name='x', + type='Tuple[int,Tuple[str,int],str]', + default="(1,('a',2),'y')" + ), + TypedArgSig(name='y', type='int', default='4'), + ] + ) + + + def test_infer_prop_type_from_docstring(self) -> None: assert_equal(infer_prop_type_from_docstring('str: A string.'), 'str') assert_equal(infer_prop_type_from_docstring('Optional[int]: An int.'), 'Optional[int]') assert_equal(infer_prop_type_from_docstring('Tuple[int, int]: A tuple.'), From 769577a43f66e77a2a45d6be215bbaeec422df11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Sat, 26 Jan 2019 13:11:04 +0100 Subject: [PATCH 09/19] Fix PEP8 --- mypy/test/teststubgen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index cbd54f68606e..b555ad006af3 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -267,7 +267,6 @@ def test_infer_arg_sig_from_docstring(self) -> None: ] ) - def test_infer_prop_type_from_docstring(self) -> None: assert_equal(infer_prop_type_from_docstring('str: A string.'), 'str') assert_equal(infer_prop_type_from_docstring('Optional[int]: An int.'), 'Optional[int]') From c77fe633bd31e42942d0775503f6d18cbb20881a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Mon, 28 Jan 2019 23:03:17 +0100 Subject: [PATCH 10/19] Review fixes --- mypy/stubgenc.py | 27 ++--- mypy/stubutil.py | 253 +++++++++++++++++++++------------------ mypy/test/teststubgen.py | 96 +++++++-------- 3 files changed, 193 insertions(+), 183 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 7c6a45f0f176..ae44db53b183 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -11,9 +11,8 @@ from types import ModuleType from mypy.stubutil import ( - is_c_module, write_header, infer_sig_from_docstring, - infer_prop_type_from_docstring, ArgList, TypedArgSig, - infer_arg_sig_from_docstring, TypedFunctionSig + is_c_module, write_header, infer_sig_from_docstring, infer_prop_type_from_docstring, + TypedArgSig, infer_arg_sig_from_docstring, TypedFunctionSig ) @@ -303,38 +302,38 @@ def is_skipped_attribute(attr: str) -> bool: '__weakref__') # For pickling -def infer_method_sig(name: str) -> ArgList: +def infer_method_sig(name: str) -> List[TypedArgSig]: if name.startswith('__') and name.endswith('__'): name = name[2:-2] if name in ('hash', 'iter', 'next', 'sizeof', 'copy', 'deepcopy', 'reduce', 'getinitargs', 'int', 'float', 'trunc', 'complex', 'bool'): return [] if name == 'getitem': - return [TypedArgSig(name='index', type=None, default=None)] + return [TypedArgSig(name='index', type=None, default=False)] if name == 'setitem': return [ - TypedArgSig(name='index', type=None, default=None), - TypedArgSig(name='object', type=None, default=None) + TypedArgSig(name='index', type=None, default=False), + TypedArgSig(name='object', type=None, default=False) ] if name in ('delattr', 'getattr'): - return [TypedArgSig(name='name', type=None, default=None)] + return [TypedArgSig(name='name', type=None, default=False)] if name == 'setattr': return [ - TypedArgSig(name='name', type=None, default=None), - TypedArgSig(name='value', type=None, default=None) + TypedArgSig(name='name', type=None, default=False), + TypedArgSig(name='value', type=None, default=False) ] if name == 'getstate': return [] if name == 'setstate': - return [TypedArgSig(name='state', type=None, default=None)] + return [TypedArgSig(name='state', type=None, default=False)] if name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul', 'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv', 'divmod', 'rdivmod', 'pow', 'rpow'): - return [TypedArgSig(name='other', type=None, default=None)] + return [TypedArgSig(name='other', type=None, default=False)] if name in ('neg', 'pos'): return [] return [ - TypedArgSig(name='*args', type=None, default=None), - TypedArgSig(name='**kwargs', type=None, default=None) + TypedArgSig(name='*args', type=None, default=False), + TypedArgSig(name='**kwargs', type=None, default=False) ] diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 06e1e550d6b9..39f2afdaae49 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -1,4 +1,4 @@ -import enum +import contextlib import io import re import sys @@ -8,6 +8,9 @@ from typing import Optional, Tuple, Sequence, MutableSequence, List, MutableMapping, IO, NamedTuple from types import ModuleType +MYPY = False +if MYPY: + from typing_extensions import Final # Type Alias for Signatures Sig = Tuple[str, str] @@ -15,18 +18,135 @@ TypedArgSig = NamedTuple('TypedArgSig', [ ('name', str), ('type', Optional[str]), - ('default', Optional[str]) + ('default', bool) ]) -ArgList = List[TypedArgSig] TypedFunctionSig = NamedTuple('TypedFunctionSig', [ ('name', str), - ('args', ArgList), + ('args', List[TypedArgSig]), ('ret_type', str) ]) +STATE_INIT = 1 # type: Final +STATE_FUNCTION_NAME = 2 # type: Final +STATE_ARGUMENT_LIST = 3 # type: Final +STATE_ARGUMENT_TYPE = 4 # type: Final +STATE_ARGUMENT_DEFAULT = 5 # type: Final +STATE_RETURN_VALUE = 6 # type: Final +STATE_OPEN_BRACKET = 7 # type: Final + + +class DocStringParser: + def __init__(self, function_name: str) -> None: + self.function_name = function_name + self.state = [STATE_INIT] + self.accumulator = "" + self.arg_type = None # type: Optional[str] + self.arg_name = "" + self.arg_default = None + self.ret_type = "Any" + self.found = False + self.args = [] # type: List[TypedArgSig] + self.signatures = [] # type: List[TypedFunctionSig] + + def add_token(self, token: tokenize.TokenInfo) -> None: + if (token.type == tokenize.NAME and token.string == self.function_name and + self.state[-1] == STATE_INIT): + self.state.append(STATE_FUNCTION_NAME) + + elif (token.type == tokenize.OP and token.string == '(' and + self.state[-1] == STATE_FUNCTION_NAME): + self.state.pop() + self.accumulator = "" + self.found = True + self.state.append(STATE_ARGUMENT_LIST) + + elif self.state[-1] == STATE_FUNCTION_NAME: + # reset state, function name not followed by '(' + self.state.pop() + + elif (token.type == tokenize.OP and token.string in ('[', '(', '{') and + self.state[-1] != STATE_INIT): + self.accumulator += token.string + self.state.append(STATE_OPEN_BRACKET) + + elif (token.type == tokenize.OP and token.string in (']', ')', '}') and + self.state[-1] == STATE_OPEN_BRACKET): + self.accumulator += token.string + self.state.pop() + + elif (token.type == tokenize.OP and token.string == ':' and + self.state[-1] == STATE_ARGUMENT_LIST): + self.arg_name = self.accumulator + self.accumulator = "" + self.state.append(STATE_ARGUMENT_TYPE) + + elif (token.type == tokenize.OP and token.string == '=' and + self.state[-1] in (STATE_ARGUMENT_LIST, STATE_ARGUMENT_TYPE)): + if self.state[-1] == STATE_ARGUMENT_TYPE: + self.arg_type = self.accumulator + self.state.pop() + else: + self.arg_name = self.accumulator + self.accumulator = "" + self.state.append(STATE_ARGUMENT_DEFAULT) + + elif (token.type == tokenize.OP and token.string in (',', ')') and + self.state[-1] in (STATE_ARGUMENT_LIST, STATE_ARGUMENT_DEFAULT, + STATE_ARGUMENT_TYPE)): + if self.state[-1] == STATE_ARGUMENT_DEFAULT: + self.arg_default = self.accumulator + self.state.pop() + elif self.state[-1] == STATE_ARGUMENT_TYPE: + self.arg_type = self.accumulator + self.state.pop() + elif self.state[-1] == STATE_ARGUMENT_LIST: + self.arg_name = self.accumulator + + if token.string == ')': + self.state.pop() + self.args.append(TypedArgSig(name=self.arg_name, type=self.arg_type, + default=bool(self.arg_default))) + self.arg_name = "" + self.arg_type = None + self.arg_default = None + self.accumulator = "" + + elif token.type == tokenize.OP and token.string == '->' and self.state[-1] == STATE_INIT: + self.accumulator = "" + self.state.append(STATE_RETURN_VALUE) + + # ENDMAKER is necessary for python 3.4 and 3.5 + elif (token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and + self.state[-1] in (STATE_INIT, STATE_RETURN_VALUE)): + if self.state[-1] == STATE_RETURN_VALUE: + self.ret_type = self.accumulator + self.accumulator = "" + self.state.pop() + + if self.found: + self.signatures.append(TypedFunctionSig(name=self.function_name, args=self.args, + ret_type=self.ret_type)) + self.found = False + self.args = [] + self.ret_type = 'Any' + # leave state as INIT + else: + self.accumulator += token.string + + def get_signatures(self) -> List[TypedFunctionSig]: + def has_arg(name: str, signature: TypedFunctionSig) -> bool: + return any(x.name == name for x in signature.args) + + def args_kwargs(signature: TypedFunctionSig) -> bool: + return has_arg('*args', signature) and has_arg('**kwargs', signature) + + # Move functions with (*args, **kwargs) in their signature to last place + return list(sorted(self.signatures, key=lambda x: 1 if args_kwargs(x) else 0)) + + def parse_signature(sig: str) -> Optional[Tuple[str, List[str], List[str]]]: @@ -123,124 +243,29 @@ def write_header(file: IO[str], module_name: Optional[str] = None, '# NOTE: This dynamically typed stub was automatically generated by stubgen.\n\n') -class State(enum.Enum): - INIT = 1 - FUNCTION_NAME = 2 - ARGUMENT_LIST = 3 - ARGUMENT_TYPE = 4 - ARGUMENT_DEFAULT = 5 - RETURN_VALUE = 6 - OPEN_BRACKET = 7 +def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunctionSig]]: + """Concert function signature to list of TypedFunctionSig + Looks for function signatures of function in docstring. Returns empty list, when no signature + is found, one signature in typical case, multiple signatures, if docstring specifies multiple + signatures for overload functions. -def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunctionSig]]: + Arguments: + * docstr: docstring + * name: name of function for which signatures are to be found + """ if not docstr: return None - state = [State.INIT] - accumulator = "" - arg_type = None - arg_name = "" - arg_default = None - ret_type = "Any" - found = False - args = [] # type: List[TypedArgSig] - signatures = [] # type: List[TypedFunctionSig] - try: + state = DocStringParser(name) + with contextlib.suppress(tokenize.TokenError): for token in tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline): - if token.type == tokenize.NAME and token.string == name and state[-1] == State.INIT: - state.append(State.FUNCTION_NAME) - - elif (token.type == tokenize.OP and token.string == '(' and - state[-1] == State.FUNCTION_NAME): - state.pop() - accumulator = "" - found = True - state.append(State.ARGUMENT_LIST) - - elif state[-1] == State.FUNCTION_NAME: - # reset state, function name not followed by '(' - state.pop() - - elif (token.type == tokenize.OP and token.string in ('[', '(', '{') and - state[-1] != State.INIT): - accumulator += token.string - state.append(State.OPEN_BRACKET) - - elif (token.type == tokenize.OP and token.string in (']', ')', '}') and - state[-1] == State.OPEN_BRACKET): - accumulator += token.string - state.pop() - - elif (token.type == tokenize.OP and token.string == ':' and - state[-1] == State.ARGUMENT_LIST): - arg_name = accumulator - accumulator = "" - state.append(State.ARGUMENT_TYPE) - - elif (token.type == tokenize.OP and token.string == '=' and - state[-1] in (State.ARGUMENT_LIST, State.ARGUMENT_TYPE)): - if state[-1] == State.ARGUMENT_TYPE: - arg_type = accumulator - state.pop() - else: - arg_name = accumulator - accumulator = "" - state.append(State.ARGUMENT_DEFAULT) - - elif (token.type == tokenize.OP and token.string in (',', ')') and - state[-1] in (State.ARGUMENT_LIST, State.ARGUMENT_DEFAULT, State.ARGUMENT_TYPE)): - if state[-1] == State.ARGUMENT_DEFAULT: - arg_default = accumulator - state.pop() - elif state[-1] == State.ARGUMENT_TYPE: - arg_type = accumulator - state.pop() - elif state[-1] == State.ARGUMENT_LIST: - arg_name = accumulator - - if token.string == ')': - state.pop() - args.append(TypedArgSig(name=arg_name, type=arg_type, default=arg_default)) - arg_name = "" - arg_type = None - arg_default = None - accumulator = "" - - elif token.type == tokenize.OP and token.string == '->' and state[-1] == State.INIT: - accumulator = "" - state.append(State.RETURN_VALUE) - - # ENDMAKER is necessary for python 3.4 and 3.5 - elif (token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and - state[-1] in (State.INIT, State.RETURN_VALUE)): - if state[-1] == State.RETURN_VALUE: - ret_type = accumulator - accumulator = "" - state.pop() - - if found: - signatures.append(TypedFunctionSig(name=name, args=args, ret_type=ret_type)) - found = False - args = [] - ret_type = 'Any' - # leave state as INIT - else: - accumulator += token.string + state.add_token(token) + return state.get_signatures() - return signatures - except tokenize.TokenError: - # return as much as collected - return signatures - -def infer_arg_sig_from_docstring(docstr: str) -> ArgList: - """ - convert signature in form of "(self: TestClass, arg0: str='ada')" to ArgList - - :param docstr: - :return: ArgList with infered argument names and its types - """ +def infer_arg_sig_from_docstring(docstr: str) -> List[TypedArgSig]: + """Convert signature in form of "(self: TestClass, arg0: str='ada')" to List[TypedArgList].""" ret = infer_sig_from_docstring("stub" + docstr, "stub") if ret: return ret[0].args diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index b555ad006af3..31de2a7406b8 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -108,7 +108,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x) - y', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type=None, default=None)], + args=[TypedArgSig(name='x', type=None, default=False)], ret_type='Any' )] ) @@ -118,8 +118,8 @@ def test_infer_sig_from_docstring(self) -> None: [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x', type=None, default=None), - TypedArgSig(name='Y_a', type=None, default='None') + TypedArgSig(name='x', type=None, default=False), + TypedArgSig(name='Y_a', type=None, default=True) ], ret_type='Any' )] @@ -129,8 +129,8 @@ def test_infer_sig_from_docstring(self) -> None: [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x', type=None, default=None), - TypedArgSig(name='Y_a', type=None, default='3') + TypedArgSig(name='x', type=None, default=False), + TypedArgSig(name='Y_a', type=None, default=True) ], ret_type='Any' )] @@ -141,8 +141,8 @@ def test_infer_sig_from_docstring(self) -> None: [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x', type=None, default=None), - TypedArgSig(name='Y_a', type=None, default='[1,2,3]') + TypedArgSig(name='x', type=None, default=False), + TypedArgSig(name='Y_a', type=None, default=True) ], ret_type='Any' )] @@ -154,7 +154,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x=z(y))', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type=None, default='z(y)')], + args=[TypedArgSig(name='x', type=None, default=True)], ret_type='Any' )] ) @@ -164,7 +164,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: int)', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default=None)], + args=[TypedArgSig(name='x', type='int', default=False)], ret_type='Any' )] ) @@ -172,7 +172,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: int=3)', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default='3')], + args=[TypedArgSig(name='x', type='int', default=True)], ret_type='Any' )] ) @@ -180,7 +180,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: int=3) -> int', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default='3')], + args=[TypedArgSig(name='x', type='int', default=True)], ret_type='int' )] ) @@ -188,7 +188,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: int=3) -> int \n', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default='3')], + args=[TypedArgSig(name='x', type='int', default=True)], ret_type='int' )] ) @@ -196,7 +196,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='Tuple[int,str]', default=None)], + args=[TypedArgSig(name='x', type='Tuple[int,str]', default=False)], ret_type='str' )] ) @@ -206,8 +206,8 @@ def test_infer_sig_from_docstring(self) -> None: [TypedFunctionSig( name='func', args=[ - TypedArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=None), - TypedArgSig(name='y', type='int', default=None), + TypedArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=False), + TypedArgSig(name='y', type='int', default=False), ], ret_type='str' )] @@ -216,7 +216,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: foo.bar)', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='foo.bar', default=None)], + args=[TypedArgSig(name='x', type='foo.bar', default=False)], ret_type='Any' )] ) @@ -225,7 +225,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: list=[1,2,[3,4]])', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='list', default='[1,2,[3,4]]')], + args=[TypedArgSig(name='x', type='list', default=True)], ret_type='Any' )] ) @@ -234,7 +234,7 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x: str="nasty[")', 'func'), [TypedFunctionSig( name='func', - args=[TypedArgSig(name='x', type='str', default='"nasty["')], + args=[TypedArgSig(name='x', type='str', default=True)], ret_type='Any' )] ) @@ -248,8 +248,8 @@ def test_infer_arg_sig_from_docstring(self) -> None: assert_equal( infer_arg_sig_from_docstring("(*args, **kwargs)"), [ - TypedArgSig(name='*args', type=None, default=None), - TypedArgSig(name='**kwargs', type=None, default=None), + TypedArgSig(name='*args', type=None, default=False), + TypedArgSig(name='**kwargs', type=None, default=False), ] ) @@ -261,9 +261,9 @@ def test_infer_arg_sig_from_docstring(self) -> None: TypedArgSig( name='x', type='Tuple[int,Tuple[str,int],str]', - default="(1,('a',2),'y')" + default=True ), - TypedArgSig(name='y', type='int', default='4'), + TypedArgSig(name='y', type='int', default=True), ] ) @@ -368,20 +368,20 @@ def test_infer_hash_sig(self) -> None: def test_infer_getitem_sig(self) -> None: assert_equal(infer_method_sig('__getitem__'), [TypedArgSig( - name='index', type=None, default=None + name='index', type=None, default=False )]) def test_infer_setitem_sig(self) -> None: assert_equal(infer_method_sig('__setitem__'), [ - TypedArgSig(name='index', type=None, default=None), - TypedArgSig(name='object', type=None, default=None) + TypedArgSig(name='index', type=None, default=False), + TypedArgSig(name='object', type=None, default=False) ]) def test_infer_binary_op_sig(self) -> None: for op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul'): assert_equal(infer_method_sig('__%s__' % op), [TypedArgSig( - name='other', type=None, default=None + name='other', type=None, default=False )]) def test_infer_unary_op_sig(self) -> None: @@ -458,17 +458,15 @@ def test(self, arg0: str) -> None: mod = ModuleType(TestClass.__module__, '') generate_c_function_stub(mod, 'test', TestClass.test, output, imports, self_var='self', class_name='TestClass') - assert_equal(output, [ - 'def test(self, arg0: int) -> Any: ...' - ]) + assert_equal(output, ['def test(self, arg0: int) -> Any: ...']) assert_equal(imports, []) def test_generate_c_function_other_module_arg(self) -> None: """ - Test that if argument references type from other module, module will be imported + Test that if argument references type from other module, module will be imported. """ - # provide different type in python spec than in docstring to make sure, that docstring - # information is used + # Provide different type in python spec than in docstring to make sure, that docstring + # information is used. def test(arg0: str) -> None: """ test(arg0: argparse.Action) @@ -478,17 +476,13 @@ def test(arg0: str) -> None: imports = [] # type: List[str] mod = ModuleType(self.__module__, '') generate_c_function_stub(mod, 'test', test, output, imports) - assert_equal(output, [ - 'def test(arg0: argparse.Action) -> Any: ...' - ]) - assert_equal(imports, [ - 'import argparse' - ]) + assert_equal(output, ['def test(arg0: argparse.Action) -> Any: ...']) + assert_equal(imports, ['import argparse']) def test_generate_c_function_same_module_arg(self) -> None: """ Test that if argument references type from same module but using full path, no module will - be imported, and type specification will be striped to local reference + be imported, and type specification will be striped to local reference. """ # provide different type in python spec than in docstring to make sure, that docstring # information is used @@ -501,14 +495,12 @@ def test(arg0: str) -> None: imports = [] # type: List[str] mod = ModuleType('argparse', '') generate_c_function_stub(mod, 'test', test, output, imports) - assert_equal(output, [ - 'def test(arg0: Action) -> Any: ...' - ]) + assert_equal(output, ['def test(arg0: Action) -> Any: ...']) assert_equal(imports, []) def test_generate_c_function_other_module_ret(self) -> None: """ - Test that if return type references type from other module, module will be imported + Test that if return type references type from other module, module will be imported. """ def test(arg0: str) -> None: """ @@ -519,17 +511,13 @@ def test(arg0: str) -> None: imports = [] # type: List[str] mod = ModuleType(self.__module__, '') generate_c_function_stub(mod, 'test', test, output, imports) - assert_equal(output, [ - 'def test(arg0: str) -> argparse.Action: ...' - ]) - assert_equal(imports, [ - 'import argparse' - ]) + assert_equal(output, ['def test(arg0: str) -> argparse.Action: ...']) + assert_equal(imports, ['import argparse']) def test_generate_c_function_same_module_ret(self) -> None: """ Test that if return type references type from same module but using full path, no module - will be imported, and type specification will be striped to local reference + will be imported, and type specification will be striped to local reference. """ def test(arg0: str) -> None: """ @@ -540,9 +528,7 @@ def test(arg0: str) -> None: imports = [] # type: List[str] mod = ModuleType('argparse', '') generate_c_function_stub(mod, 'test', test, output, imports) - assert_equal(output, [ - 'def test(arg0: str) -> Action: ...' - ]) + assert_equal(output, ['def test(arg0: str) -> Action: ...']) assert_equal(imports, []) def test_generate_c_type_with_overload_pybind11(self) -> None: @@ -563,12 +549,12 @@ def __init__(self, arg0: str) -> None: generate_c_function_stub(mod, '__init__', TestClass.__init__, output, imports, self_var='self', class_name='TestClass') assert_equal(output, [ - '@overload', - 'def __init__(*args, **kwargs) -> Any: ...', '@overload', 'def __init__(self, arg0: str) -> None: ...', '@overload', 'def __init__(self, arg0: str, arg1: str) -> None: ...', + '@overload', + 'def __init__(*args, **kwargs) -> Any: ...', ]) assert_equal(set(imports), { 'from typing import overload' From 824c65abaebaeae1b1e841224ac24333c5abaad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Mon, 28 Jan 2019 23:30:43 +0100 Subject: [PATCH 11/19] Fixed mypy self-check --- mypy/stubutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 39f2afdaae49..8ab895fda457 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -45,7 +45,7 @@ def __init__(self, function_name: str) -> None: self.accumulator = "" self.arg_type = None # type: Optional[str] self.arg_name = "" - self.arg_default = None + self.arg_default = None # type: Optional[str] self.ret_type = "Any" self.found = False self.args = [] # type: List[TypedArgSig] From 382946bee80e3548f94ffaaeb866fa083f55fa6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 18:18:44 +0100 Subject: [PATCH 12/19] Review fixes TypedArgSig -> ArgSig TypedFunctionSig -> FunctionSig --- mypy/stubgenc.py | 38 +++++++++---------- mypy/stubutil.py | 28 +++++++------- mypy/test/teststubgen.py | 82 ++++++++++++++++++++-------------------- 3 files changed, 74 insertions(+), 74 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index ae44db53b183..a163504b32b5 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -12,7 +12,7 @@ from mypy.stubutil import ( is_c_module, write_header, infer_sig_from_docstring, infer_prop_type_from_docstring, - TypedArgSig, infer_arg_sig_from_docstring, TypedFunctionSig + ArgSig, infer_arg_sig_from_docstring, FunctionSig ) @@ -125,20 +125,20 @@ def generate_c_function_stub(module: ModuleType, if (name in ('__new__', '__init__') and name not in sigs and class_name and class_name in class_sigs): - inferred = [TypedFunctionSig(name=name, - args=infer_arg_sig_from_docstring(class_sigs[class_name]), - ret_type=ret_type)] # type: Optional[List[TypedFunctionSig]] + inferred = [FunctionSig(name=name, + args=infer_arg_sig_from_docstring(class_sigs[class_name]), + ret_type=ret_type)] # type: Optional[List[FunctionSig]] else: docstr = getattr(obj, '__doc__', None) inferred = infer_sig_from_docstring(docstr, name) if not inferred: if class_name and name not in sigs: - inferred = [TypedFunctionSig(name, args=infer_method_sig(name), ret_type=ret_type)] + inferred = [FunctionSig(name, args=infer_method_sig(name), ret_type=ret_type)] else: - inferred = [TypedFunctionSig(name=name, - args=infer_arg_sig_from_docstring( + inferred = [FunctionSig(name=name, + args=infer_arg_sig_from_docstring( sigs.get(name, '(*args, **kwargs)')), - ret_type=ret_type)] + ret_type=ret_type)] is_overloaded = len(inferred) > 1 if inferred else False if is_overloaded: @@ -302,38 +302,38 @@ def is_skipped_attribute(attr: str) -> bool: '__weakref__') # For pickling -def infer_method_sig(name: str) -> List[TypedArgSig]: +def infer_method_sig(name: str) -> List[ArgSig]: if name.startswith('__') and name.endswith('__'): name = name[2:-2] if name in ('hash', 'iter', 'next', 'sizeof', 'copy', 'deepcopy', 'reduce', 'getinitargs', 'int', 'float', 'trunc', 'complex', 'bool'): return [] if name == 'getitem': - return [TypedArgSig(name='index', type=None, default=False)] + return [ArgSig(name='index', type=None, default=False)] if name == 'setitem': return [ - TypedArgSig(name='index', type=None, default=False), - TypedArgSig(name='object', type=None, default=False) + ArgSig(name='index', type=None, default=False), + ArgSig(name='object', type=None, default=False) ] if name in ('delattr', 'getattr'): - return [TypedArgSig(name='name', type=None, default=False)] + return [ArgSig(name='name', type=None, default=False)] if name == 'setattr': return [ - TypedArgSig(name='name', type=None, default=False), - TypedArgSig(name='value', type=None, default=False) + ArgSig(name='name', type=None, default=False), + ArgSig(name='value', type=None, default=False) ] if name == 'getstate': return [] if name == 'setstate': - return [TypedArgSig(name='state', type=None, default=False)] + return [ArgSig(name='state', type=None, default=False)] if name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul', 'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv', 'divmod', 'rdivmod', 'pow', 'rpow'): - return [TypedArgSig(name='other', type=None, default=False)] + return [ArgSig(name='other', type=None, default=False)] if name in ('neg', 'pos'): return [] return [ - TypedArgSig(name='*args', type=None, default=False), - TypedArgSig(name='**kwargs', type=None, default=False) + ArgSig(name='*args', type=None, default=False), + ArgSig(name='**kwargs', type=None, default=False) ] diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 8ab895fda457..085d6127b643 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -15,16 +15,16 @@ # Type Alias for Signatures Sig = Tuple[str, str] -TypedArgSig = NamedTuple('TypedArgSig', [ +ArgSig = NamedTuple('ArgSig', [ ('name', str), ('type', Optional[str]), ('default', bool) ]) -TypedFunctionSig = NamedTuple('TypedFunctionSig', [ +FunctionSig = NamedTuple('FunctionSig', [ ('name', str), - ('args', List[TypedArgSig]), + ('args', List[ArgSig]), ('ret_type', str) ]) @@ -48,8 +48,8 @@ def __init__(self, function_name: str) -> None: self.arg_default = None # type: Optional[str] self.ret_type = "Any" self.found = False - self.args = [] # type: List[TypedArgSig] - self.signatures = [] # type: List[TypedFunctionSig] + self.args = [] # type: List[ArgSig] + self.signatures = [] # type: List[FunctionSig] def add_token(self, token: tokenize.TokenInfo) -> None: if (token.type == tokenize.NAME and token.string == self.function_name and @@ -107,8 +107,8 @@ def add_token(self, token: tokenize.TokenInfo) -> None: if token.string == ')': self.state.pop() - self.args.append(TypedArgSig(name=self.arg_name, type=self.arg_type, - default=bool(self.arg_default))) + self.args.append(ArgSig(name=self.arg_name, type=self.arg_type, + default=bool(self.arg_default))) self.arg_name = "" self.arg_type = None self.arg_default = None @@ -127,8 +127,8 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.state.pop() if self.found: - self.signatures.append(TypedFunctionSig(name=self.function_name, args=self.args, - ret_type=self.ret_type)) + self.signatures.append(FunctionSig(name=self.function_name, args=self.args, + ret_type=self.ret_type)) self.found = False self.args = [] self.ret_type = 'Any' @@ -136,11 +136,11 @@ def add_token(self, token: tokenize.TokenInfo) -> None: else: self.accumulator += token.string - def get_signatures(self) -> List[TypedFunctionSig]: - def has_arg(name: str, signature: TypedFunctionSig) -> bool: + def get_signatures(self) -> List[FunctionSig]: + def has_arg(name: str, signature: FunctionSig) -> bool: return any(x.name == name for x in signature.args) - def args_kwargs(signature: TypedFunctionSig) -> bool: + def args_kwargs(signature: FunctionSig) -> bool: return has_arg('*args', signature) and has_arg('**kwargs', signature) # Move functions with (*args, **kwargs) in their signature to last place @@ -243,7 +243,7 @@ def write_header(file: IO[str], module_name: Optional[str] = None, '# NOTE: This dynamically typed stub was automatically generated by stubgen.\n\n') -def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunctionSig]]: +def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[FunctionSig]]: """Concert function signature to list of TypedFunctionSig Looks for function signatures of function in docstring. Returns empty list, when no signature @@ -264,7 +264,7 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[TypedFunct return state.get_signatures() -def infer_arg_sig_from_docstring(docstr: str) -> List[TypedArgSig]: +def infer_arg_sig_from_docstring(docstr: str) -> List[ArgSig]: """Convert signature in form of "(self: TestClass, arg0: str='ada')" to List[TypedArgList].""" ret = infer_sig_from_docstring("stub" + docstr, "stub") if ret: diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 31de2a7406b8..9653135c6f1a 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -17,7 +17,7 @@ from mypy.stubgenc import generate_c_type_stub, infer_method_sig, generate_c_function_stub from mypy.stubutil import ( parse_signature, parse_all_signatures, build_signature, find_unique_signatures, - infer_sig_from_docstring, infer_prop_type_from_docstring, TypedFunctionSig, TypedArgSig, + infer_sig_from_docstring, infer_prop_type_from_docstring, FunctionSig, ArgSig, infer_arg_sig_from_docstring ) @@ -106,31 +106,31 @@ def test_find_unique_signatures(self) -> None: def test_infer_sig_from_docstring(self) -> None: assert_equal( infer_sig_from_docstring('\nfunc(x) - y', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type=None, default=False)], + args=[ArgSig(name='x', type=None, default=False)], ret_type='Any' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=None)', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', args=[ - TypedArgSig(name='x', type=None, default=False), - TypedArgSig(name='Y_a', type=None, default=True) + ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True) ], ret_type='Any' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=3)', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', args=[ - TypedArgSig(name='x', type=None, default=False), - TypedArgSig(name='Y_a', type=None, default=True) + ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True) ], ret_type='Any' )] @@ -138,11 +138,11 @@ def test_infer_sig_from_docstring(self) -> None: assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=[1, 2, 3])', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', args=[ - TypedArgSig(name='x', type=None, default=False), - TypedArgSig(name='Y_a', type=None, default=True) + ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True) ], ret_type='Any' )] @@ -152,9 +152,9 @@ def test_infer_sig_from_docstring(self) -> None: assert_equal(infer_sig_from_docstring('\nfunc(x, y', 'func'), []) assert_equal( infer_sig_from_docstring('\nfunc(x=z(y))', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type=None, default=True)], + args=[ArgSig(name='x', type=None, default=True)], ret_type='Any' )] ) @@ -162,79 +162,79 @@ def test_infer_sig_from_docstring(self) -> None: # try to infer signature from type annotation assert_equal( infer_sig_from_docstring('\nfunc(x: int)', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default=False)], + args=[ArgSig(name='x', type='int', default=False)], ret_type='Any' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3)', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default=True)], + args=[ArgSig(name='x', type='int', default=True)], ret_type='Any' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3) -> int', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default=True)], + args=[ArgSig(name='x', type='int', default=True)], ret_type='int' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3) -> int \n', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='int', default=True)], + args=[ArgSig(name='x', type='int', default=True)], ret_type='int' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='Tuple[int,str]', default=False)], + args=[ArgSig(name='x', type='Tuple[int,str]', default=False)], ret_type='str' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, Tuple[str, int], str], y: int) -> str', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', args=[ - TypedArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=False), - TypedArgSig(name='y', type='int', default=False), + ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=False), + ArgSig(name='y', type='int', default=False), ], ret_type='str' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: foo.bar)', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='foo.bar', default=False)], + args=[ArgSig(name='x', type='foo.bar', default=False)], ret_type='Any' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: list=[1,2,[3,4]])', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='list', default=True)], + args=[ArgSig(name='x', type='list', default=True)], ret_type='Any' )] ) assert_equal( infer_sig_from_docstring('\nfunc(x: str="nasty[")', 'func'), - [TypedFunctionSig( + [FunctionSig( name='func', - args=[TypedArgSig(name='x', type='str', default=True)], + args=[ArgSig(name='x', type='str', default=True)], ret_type='Any' )] ) @@ -248,8 +248,8 @@ def test_infer_arg_sig_from_docstring(self) -> None: assert_equal( infer_arg_sig_from_docstring("(*args, **kwargs)"), [ - TypedArgSig(name='*args', type=None, default=False), - TypedArgSig(name='**kwargs', type=None, default=False), + ArgSig(name='*args', type=None, default=False), + ArgSig(name='**kwargs', type=None, default=False), ] ) @@ -258,12 +258,12 @@ def test_infer_arg_sig_from_docstring(self) -> None: "(x: Tuple[int, Tuple[str, int], str]=(1, ('a', 2), 'y'), y: int=4)" ), [ - TypedArgSig( + ArgSig( name='x', type='Tuple[int,Tuple[str,int],str]', default=True ), - TypedArgSig(name='y', type='int', default=True), + ArgSig(name='y', type='int', default=True), ] ) @@ -367,20 +367,20 @@ def test_infer_hash_sig(self) -> None: assert_equal(infer_method_sig('__hash__'), []) def test_infer_getitem_sig(self) -> None: - assert_equal(infer_method_sig('__getitem__'), [TypedArgSig( + assert_equal(infer_method_sig('__getitem__'), [ArgSig( name='index', type=None, default=False )]) def test_infer_setitem_sig(self) -> None: assert_equal(infer_method_sig('__setitem__'), [ - TypedArgSig(name='index', type=None, default=False), - TypedArgSig(name='object', type=None, default=False) + ArgSig(name='index', type=None, default=False), + ArgSig(name='object', type=None, default=False) ]) def test_infer_binary_op_sig(self) -> None: for op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul'): - assert_equal(infer_method_sig('__%s__' % op), [TypedArgSig( + assert_equal(infer_method_sig('__%s__' % op), [ArgSig( name='other', type=None, default=False )]) From fb7ad2dc1729390789bf81174733839c34490b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 18:27:18 +0100 Subject: [PATCH 13/19] Fix docstring style --- mypy/test/teststubgen.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 9653135c6f1a..a1657f459993 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -462,9 +462,7 @@ def test(self, arg0: str) -> None: assert_equal(imports, []) def test_generate_c_function_other_module_arg(self) -> None: - """ - Test that if argument references type from other module, module will be imported. - """ + """Test that if argument references type from other module, module will be imported.""" # Provide different type in python spec than in docstring to make sure, that docstring # information is used. def test(arg0: str) -> None: From eebb0e1cef4bc0fa3070ca1c5d07effca126b743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 18:38:16 +0100 Subject: [PATCH 14/19] Review fixes Shorten multiline lists. --- mypy/stubgenc.py | 12 ++++-------- mypy/test/teststubgen.py | 40 ++++++++++++---------------------------- 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index a163504b32b5..106e65f44337 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -311,17 +311,13 @@ def infer_method_sig(name: str) -> List[ArgSig]: if name == 'getitem': return [ArgSig(name='index', type=None, default=False)] if name == 'setitem': - return [ - ArgSig(name='index', type=None, default=False), - ArgSig(name='object', type=None, default=False) - ] + return [ArgSig(name='index', type=None, default=False), + ArgSig(name='object', type=None, default=False)] if name in ('delattr', 'getattr'): return [ArgSig(name='name', type=None, default=False)] if name == 'setattr': - return [ - ArgSig(name='name', type=None, default=False), - ArgSig(name='value', type=None, default=False) - ] + return [ArgSig(name='name', type=None, default=False), + ArgSig(name='value', type=None, default=False)] if name == 'getstate': return [] if name == 'setstate': diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index a1657f459993..1f1f12f6b0a6 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -117,10 +117,8 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x, Y_a=None)', 'func'), [FunctionSig( name='func', - args=[ - ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True) - ], + args=[ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True)], ret_type='Any' )] ) @@ -128,10 +126,8 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x, Y_a=3)', 'func'), [FunctionSig( name='func', - args=[ - ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True) - ], + args=[ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True)], ret_type='Any' )] ) @@ -140,10 +136,8 @@ def test_infer_sig_from_docstring(self) -> None: infer_sig_from_docstring('\nfunc(x, Y_a=[1, 2, 3])', 'func'), [FunctionSig( name='func', - args=[ - ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True) - ], + args=[ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True)], ret_type='Any' )] ) @@ -205,10 +199,8 @@ def test_infer_sig_from_docstring(self) -> None: 'func'), [FunctionSig( name='func', - args=[ - ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=False), - ArgSig(name='y', type='int', default=False), - ], + args=[ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=False), + ArgSig(name='y', type='int', default=False)], ret_type='str' )] ) @@ -247,24 +239,16 @@ def test_infer_sig_from_docstring(self) -> None: def test_infer_arg_sig_from_docstring(self) -> None: assert_equal( infer_arg_sig_from_docstring("(*args, **kwargs)"), - [ - ArgSig(name='*args', type=None, default=False), - ArgSig(name='**kwargs', type=None, default=False), - ] + [ArgSig(name='*args', type=None, default=False), + ArgSig(name='**kwargs', type=None, default=False)] ) assert_equal( infer_arg_sig_from_docstring( "(x: Tuple[int, Tuple[str, int], str]=(1, ('a', 2), 'y'), y: int=4)" ), - [ - ArgSig( - name='x', - type='Tuple[int,Tuple[str,int],str]', - default=True - ), - ArgSig(name='y', type='int', default=True), - ] + [ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=True), + ArgSig(name='y', type='int', default=True)] ) def test_infer_prop_type_from_docstring(self) -> None: From cd02f06a9931d15275e2cbc5ff13a8a935fd09d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 18:43:28 +0100 Subject: [PATCH 15/19] Shorten lists --- mypy/test/teststubgen.py | 121 +++++++++++++++------------------------ 1 file changed, 47 insertions(+), 74 deletions(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 1f1f12f6b0a6..b403a3a6de9b 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -106,129 +106,102 @@ def test_find_unique_signatures(self) -> None: def test_infer_sig_from_docstring(self) -> None: assert_equal( infer_sig_from_docstring('\nfunc(x) - y', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type=None, default=False)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type=None, default=False)], + ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=None)', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True)], + ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=3)', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True)], + ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=[1, 2, 3])', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type=None, default=False), + ArgSig(name='Y_a', type=None, default=True)], + ret_type='Any')] ) assert_equal(infer_sig_from_docstring('\nafunc(x) - y', 'func'), []) assert_equal(infer_sig_from_docstring('\nfunc(x, y', 'func'), []) assert_equal( infer_sig_from_docstring('\nfunc(x=z(y))', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type=None, default=True)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type=None, default=True)], + ret_type='Any')] ) assert_equal(infer_sig_from_docstring('\nfunc x', 'func'), []) # try to infer signature from type annotation assert_equal( infer_sig_from_docstring('\nfunc(x: int)', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='int', default=False)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='int', default=False)], + ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3)', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='int', default=True)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='int', default=True)], + ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3) -> int', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='int', default=True)], - ret_type='int' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='int', default=True)], + ret_type='int')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3) -> int \n', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='int', default=True)], - ret_type='int' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='int', default=True)], + ret_type='int')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='Tuple[int,str]', default=False)], - ret_type='str' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='Tuple[int,str]', default=False)], + ret_type='str')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, Tuple[str, int], str], y: int) -> str', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', default=False), - ArgSig(name='y', type='int', default=False)], - ret_type='str' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', + default=False), + ArgSig(name='y', type='int', default=False)], + ret_type='str')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: foo.bar)', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='foo.bar', default=False)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='foo.bar', default=False)], + ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: list=[1,2,[3,4]])', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='list', default=True)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='list', default=True)], + ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: str="nasty[")', 'func'), - [FunctionSig( - name='func', - args=[ArgSig(name='x', type='str', default=True)], - ret_type='Any' - )] + [FunctionSig(name='func', + args=[ArgSig(name='x', type='str', default=True)], + ret_type='Any')] ) assert_equal( From e0ace0f676b204e8ab870af1b6d1aa803be0ad67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 19:26:53 +0100 Subject: [PATCH 16/19] fix PEP8 --- mypy/stubgenc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 106e65f44337..eb379771aeb0 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -137,7 +137,7 @@ def generate_c_function_stub(module: ModuleType, else: inferred = [FunctionSig(name=name, args=infer_arg_sig_from_docstring( - sigs.get(name, '(*args, **kwargs)')), + sigs.get(name, '(*args, **kwargs)')), ret_type=ret_type)] is_overloaded = len(inferred) > 1 if inferred else False From d4be9481a6c9ee30305cfa66cbce544ef16969e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 20:51:19 +0100 Subject: [PATCH 17/19] Review fixes. Create ArgList class instead of NamedTuple and provide defaults for type and default --- mypy/stubutil.py | 24 ++++++++++++---- mypy/test/teststubgen.py | 61 ++++++++++++++++++---------------------- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 085d6127b643..be27c23f5317 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -5,7 +5,8 @@ import os import tokenize -from typing import Optional, Tuple, Sequence, MutableSequence, List, MutableMapping, IO, NamedTuple +from typing import (Optional, Tuple, Sequence, MutableSequence, List, MutableMapping, IO, + NamedTuple, Any) from types import ModuleType MYPY = False @@ -15,11 +16,22 @@ # Type Alias for Signatures Sig = Tuple[str, str] -ArgSig = NamedTuple('ArgSig', [ - ('name', str), - ('type', Optional[str]), - ('default', bool) -]) + +class ArgSig: + def __init__(self, name: str, type: Optional[str] = None, default: bool = False): + self.name = name + self.type = type + self.default = default + + def __repr__(self) -> str: + return "ArgSig(name={}, type={}, default={})".format(repr(self.name), repr(self.type), + repr(self.default)) + + def __eq__(self, other: Any) -> bool: + if isinstance(other, ArgSig): + return (self.name == other.name and self.type == other.type and + self.default == other.default) + return False FunctionSig = NamedTuple('FunctionSig', [ diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index b403a3a6de9b..26ac3fb1db3f 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -106,49 +106,40 @@ def test_find_unique_signatures(self) -> None: def test_infer_sig_from_docstring(self) -> None: assert_equal( infer_sig_from_docstring('\nfunc(x) - y', 'func'), - [FunctionSig(name='func', - args=[ArgSig(name='x', type=None, default=False)], - ret_type='Any')] + [FunctionSig(name='func', args=[ArgSig(name='x')], ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=None)', 'func'), [FunctionSig(name='func', - args=[ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True)], + args=[ArgSig(name='x'), ArgSig(name='Y_a', default=True)], ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=3)', 'func'), [FunctionSig(name='func', - args=[ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True)], + args=[ArgSig(name='x'), ArgSig(name='Y_a', default=True)], ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x, Y_a=[1, 2, 3])', 'func'), [FunctionSig(name='func', - args=[ArgSig(name='x', type=None, default=False), - ArgSig(name='Y_a', type=None, default=True)], - ret_type='Any')] + args=[ArgSig(name='x'), ArgSig(name='Y_a', default=True)], + ret_type='Any')] ) assert_equal(infer_sig_from_docstring('\nafunc(x) - y', 'func'), []) assert_equal(infer_sig_from_docstring('\nfunc(x, y', 'func'), []) assert_equal( infer_sig_from_docstring('\nfunc(x=z(y))', 'func'), - [FunctionSig(name='func', - args=[ArgSig(name='x', type=None, default=True)], - ret_type='Any')] + [FunctionSig(name='func', args=[ArgSig(name='x', default=True)], ret_type='Any')] ) assert_equal(infer_sig_from_docstring('\nfunc x', 'func'), []) # try to infer signature from type annotation assert_equal( infer_sig_from_docstring('\nfunc(x: int)', 'func'), - [FunctionSig(name='func', - args=[ArgSig(name='x', type='int', default=False)], - ret_type='Any')] + [FunctionSig(name='func', args=[ArgSig(name='x', type='int')], ret_type='Any')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: int=3)', 'func'), @@ -171,22 +162,21 @@ def test_infer_sig_from_docstring(self) -> None: assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, str]) -> str', 'func'), [FunctionSig(name='func', - args=[ArgSig(name='x', type='Tuple[int,str]', default=False)], + args=[ArgSig(name='x', type='Tuple[int,str]')], ret_type='str')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: Tuple[int, Tuple[str, int], str], y: int) -> str', 'func'), [FunctionSig(name='func', - args=[ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]', - default=False), - ArgSig(name='y', type='int', default=False)], + args=[ArgSig(name='x', type='Tuple[int,Tuple[str,int],str]'), + ArgSig(name='y', type='int')], ret_type='str')] ) assert_equal( infer_sig_from_docstring('\nfunc(x: foo.bar)', 'func'), [FunctionSig(name='func', - args=[ArgSig(name='x', type='foo.bar', default=False)], + args=[ArgSig(name='x', type='foo.bar')], ret_type='Any')] ) @@ -212,8 +202,7 @@ def test_infer_sig_from_docstring(self) -> None: def test_infer_arg_sig_from_docstring(self) -> None: assert_equal( infer_arg_sig_from_docstring("(*args, **kwargs)"), - [ArgSig(name='*args', type=None, default=False), - ArgSig(name='**kwargs', type=None, default=False)] + [ArgSig(name='*args'), ArgSig(name='**kwargs')] ) assert_equal( @@ -324,22 +313,16 @@ def test_infer_hash_sig(self) -> None: assert_equal(infer_method_sig('__hash__'), []) def test_infer_getitem_sig(self) -> None: - assert_equal(infer_method_sig('__getitem__'), [ArgSig( - name='index', type=None, default=False - )]) + assert_equal(infer_method_sig('__getitem__'), [ArgSig(name='index')]) def test_infer_setitem_sig(self) -> None: - assert_equal(infer_method_sig('__setitem__'), [ - ArgSig(name='index', type=None, default=False), - ArgSig(name='object', type=None, default=False) - ]) + assert_equal(infer_method_sig('__setitem__'), + [ArgSig(name='index'), ArgSig(name='object')]) def test_infer_binary_op_sig(self) -> None: for op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul'): - assert_equal(infer_method_sig('__%s__' % op), [ArgSig( - name='other', type=None, default=False - )]) + assert_equal(infer_method_sig('__%s__' % op), [ArgSig(name='other')]) def test_infer_unary_op_sig(self) -> None: for op in ('neg', 'pos'): @@ -514,3 +497,15 @@ def __init__(self, arg0: str) -> None: assert_equal(set(imports), { 'from typing import overload' }) + + +class ArgSigSuite(Suite): + def test_repr(self): + assert_equal(repr(ArgSig(name='asd"dsa')), + "ArgSig(name='asd\"dsa', type=None, default=False)") + assert_equal(repr(ArgSig(name="asd'dsa")), + 'ArgSig(name="asd\'dsa", type=None, default=False)') + assert_equal(repr(ArgSig("func", 'str')), + "ArgSig(name='func', type='str', default=False)") + assert_equal(repr(ArgSig("func", 'str', default=True)), + "ArgSig(name='func', type='str', default=True)") From e42097bfe5161a5ea2ef8c5a2c3337cf24f674d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 20:57:08 +0100 Subject: [PATCH 18/19] Fix self-check --- mypy/test/teststubgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 26ac3fb1db3f..f4083c8cef38 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -500,7 +500,7 @@ def __init__(self, arg0: str) -> None: class ArgSigSuite(Suite): - def test_repr(self): + def test_repr(self) -> None: assert_equal(repr(ArgSig(name='asd"dsa')), "ArgSig(name='asd\"dsa', type=None, default=False)") assert_equal(repr(ArgSig(name="asd'dsa")), From 8082b308519965ccd0e9ff52ba4e64092cd33440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 29 Jan 2019 21:08:43 +0100 Subject: [PATCH 19/19] Remove default values from ArgSig --- mypy/stubgenc.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index eb379771aeb0..6e87c5b1f96c 100644 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -309,27 +309,27 @@ def infer_method_sig(name: str) -> List[ArgSig]: 'int', 'float', 'trunc', 'complex', 'bool'): return [] if name == 'getitem': - return [ArgSig(name='index', type=None, default=False)] + return [ArgSig(name='index')] if name == 'setitem': - return [ArgSig(name='index', type=None, default=False), - ArgSig(name='object', type=None, default=False)] + return [ArgSig(name='index'), + ArgSig(name='object')] if name in ('delattr', 'getattr'): - return [ArgSig(name='name', type=None, default=False)] + return [ArgSig(name='name')] if name == 'setattr': - return [ArgSig(name='name', type=None, default=False), - ArgSig(name='value', type=None, default=False)] + return [ArgSig(name='name'), + ArgSig(name='value')] if name == 'getstate': return [] if name == 'setstate': - return [ArgSig(name='state', type=None, default=False)] + return [ArgSig(name='state')] if name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'add', 'radd', 'sub', 'rsub', 'mul', 'rmul', 'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv', 'divmod', 'rdivmod', 'pow', 'rpow'): - return [ArgSig(name='other', type=None, default=False)] + return [ArgSig(name='other')] if name in ('neg', 'pos'): return [] return [ - ArgSig(name='*args', type=None, default=False), - ArgSig(name='**kwargs', type=None, default=False) + ArgSig(name='*args'), + ArgSig(name='**kwargs') ]