From 9238e0647bf2f125019771be040c21c302d17f7a Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 16 Sep 2020 14:37:04 +0200 Subject: [PATCH 01/30] show in linter custom message for pre contract --- deal/linter/_extractors/common.py | 1 + deal/linter/_extractors/pre.py | 8 ++++++-- deal/linter/_rules.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/deal/linter/_extractors/common.py b/deal/linter/_extractors/common.py index 1db032db..2316a64c 100644 --- a/deal/linter/_extractors/common.py +++ b/deal/linter/_extractors/common.py @@ -38,6 +38,7 @@ class Token(NamedTuple): line: int col: int value: Optional[object] = None + # marker name or error message: marker: Optional[str] = None diff --git a/deal/linter/_extractors/pre.py b/deal/linter/_extractors/pre.py index 3725e56f..ba96e38f 100644 --- a/deal/linter/_extractors/pre.py +++ b/deal/linter/_extractors/pre.py @@ -55,8 +55,12 @@ def handle_call(expr: astroid.Call) -> Iterator[Token]: except NameError: continue if result is False or type(result) is str: - msg = format_call_args(args, kwargs) - yield Token(value=msg, line=expr.lineno, col=expr.col_offset) + yield Token( + value=format_call_args(args, kwargs), + marker=result or None, + line=expr.lineno, + col=expr.col_offset, + ) def format_call_args(args: Sequence, kwargs: dict) -> str: diff --git a/deal/linter/_rules.py b/deal/linter/_rules.py index 452517f3..353f7a3b 100644 --- a/deal/linter/_rules.py +++ b/deal/linter/_rules.py @@ -64,7 +64,7 @@ def __call__(self, func: Func, stubs: StubsManager = None) -> Iterator[Error]: for token in get_pre(body=func.body): yield Error( code=self.code, - text=self.message, + text=token.marker or self.message, value=token.value, # type: ignore row=token.line, col=token.col, From 9500746670c4414ef7e4804b8bdc4a3c7bfaabf6 Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 16 Sep 2020 14:37:12 +0200 Subject: [PATCH 02/30] +printf example --- examples/printf.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 examples/printf.py diff --git a/examples/printf.py b/examples/printf.py new file mode 100644 index 00000000..7d6a3ca4 --- /dev/null +++ b/examples/printf.py @@ -0,0 +1,35 @@ +import deal + + +def contract(template: str, *args): + import re + rex = re.compile(r'\{\:([a-z])\}') + types = {'s': str, 'd': float} + matches = rex.findall(template) + if len(matches) != len(args): + return f'expected {len(matches)} argument(s) but {len(args)} found' + for match, arg in zip(matches, args): + t = types[match[0]] + if not isinstance(arg, t): + return f'expected {t.__name__}, {type(arg).__name__} given' + return True + + +@deal.pre(contract) +def format(template: str, *args) -> str: + return template.format(*args) + + +@deal.has('io') +def example(): + # good + print(format('{:s}', 'hello')) + + # bad + print(format('{:s}')) # not enough args + print(format('{:s}', 'a', 'b')) # too many args + print(format('{:d}', 'a')) # bad type + + +if __name__ == '__main__': + print(format('{:s} {:s}', 'hello', 'world')) From 0451b0980165d9a42cfdf31d647746a34ff27230 Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 16 Sep 2020 15:35:02 +0200 Subject: [PATCH 03/30] resolve dependency on an external module --- deal/linter/_contract.py | 51 ++++++++++++++++++++++++++++++++++++---- examples/printf.py | 3 ++- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/deal/linter/_contract.py b/deal/linter/_contract.py index 9ae7cb43..8139df39 100644 --- a/deal/linter/_contract.py +++ b/deal/linter/_contract.py @@ -2,6 +2,10 @@ import ast import builtins import enum +import pdb +import sys +from distutils.sysconfig import get_python_lib +from importlib.util import find_spec from pathlib import Path # external @@ -19,21 +23,60 @@ class Category(enum.Enum): PURE = 'pure' +class cached_property: + def __init__(self, func): + self.func = func + + def __get__(self, obj, cls): + if obj is None: + return self + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value + + class Contract: - __slots__ = ('args', 'category', 'func_args') def __init__(self, args, category: Category, func_args: ast.arguments): self.args = args self.category = category self.func_args = func_args - @property + @cached_property def body(self) -> ast.AST: contract = self.args[0] # convert astroid node to ast node if hasattr(contract, 'as_string'): contract = self._resolve_name(contract) contract = ast.parse(contract.as_string()).body[0] + contract = self._resolve_deps(contract) + return contract + + @staticmethod + def _resolve_deps(contract): + # infer stdlib modules + modules = set() + stdlib_prefix = get_python_lib(standard_lib=True) + for node in ast.walk(contract): + if not isinstance(node, ast.Name): + continue + if hasattr(builtins, node.id): + continue + # C-compiled built-in + if node.id in sys.builtin_module_names: + modules.add(node.id) + continue + # stdlib + spec = find_spec(node.id) + if spec and spec.origin.startswith(stdlib_prefix): + modules.add(node.id) + continue + + # import inferred modules + for module in modules: + node: ast.Import = ast.parse('import something').body[0] + node.names[0].name = module + contract.body.insert(0, node) + return contract @staticmethod @@ -51,7 +94,7 @@ def _resolve_name(contract): # resolved into something tricky, live with it return contract # pragma: no cover - @property + @cached_property def exceptions(self) -> list: # app from ._extractors import get_name @@ -65,7 +108,7 @@ def exceptions(self) -> list: excs.append(exc) return excs - @property + @cached_property def bytecode(self): module = ast.parse(TEMPLATE) diff --git a/examples/printf.py b/examples/printf.py index 7d6a3ca4..99c5b224 100644 --- a/examples/printf.py +++ b/examples/printf.py @@ -1,8 +1,9 @@ +import re + import deal def contract(template: str, *args): - import re rex = re.compile(r'\{\:([a-z])\}') types = {'s': str, 'd': float} matches = rex.findall(template) From 239930a2154d74aac492d9882687e7368e4be7b9 Mon Sep 17 00:00:00 2001 From: Gram Date: Mon, 21 Sep 2020 15:37:28 +0200 Subject: [PATCH 04/30] find external deps for contracts --- deal/linter/_contract.py | 77 ++++++++++++++++++------------ deal/linter/_func.py | 30 +++++++++++- tests/test_linter/test_contract.py | 53 ++++++++++++++++++++ tests/test_linter/test_func.py | 19 ++++++++ 4 files changed, 148 insertions(+), 31 deletions(-) diff --git a/deal/linter/_contract.py b/deal/linter/_contract.py index 8139df39..e9ac710f 100644 --- a/deal/linter/_contract.py +++ b/deal/linter/_contract.py @@ -2,11 +2,8 @@ import ast import builtins import enum -import pdb -import sys -from distutils.sysconfig import get_python_lib -from importlib.util import find_spec from pathlib import Path +from typing import Dict, FrozenSet, Iterable # external import astroid @@ -35,11 +32,22 @@ def __get__(self, obj, cls): class Contract: - - def __init__(self, args, category: Category, func_args: ast.arguments): - self.args = args + args: tuple + category: Category + func_args: ast.arguments + context: Dict[str, ast.AST] + + def __init__( + self, + args: Iterable, + category: Category, + func_args: ast.arguments, + context: Dict[str, ast.AST] = None, + ): + self.args = tuple(args) self.category = category self.func_args = func_args + self.context = context or dict() @cached_property def body(self) -> ast.AST: @@ -48,36 +56,45 @@ def body(self) -> ast.AST: if hasattr(contract, 'as_string'): contract = self._resolve_name(contract) contract = ast.parse(contract.as_string()).body[0] - contract = self._resolve_deps(contract) return contract - @staticmethod - def _resolve_deps(contract): - # infer stdlib modules - modules = set() - stdlib_prefix = get_python_lib(standard_lib=True) - for node in ast.walk(contract): + @cached_property + def arguments(self) -> FrozenSet[str]: + """Contract function arguments names. + + Useful for resolving external dependencies. + """ + if not isinstance(self.body, (ast.FunctionDef, ast.Lambda)): + return frozenset() + args = self.body.args + result = set() + for arg in args.args: + result.add(arg.arg) + for arg in args.kwonlyargs: + result.add(arg.arg) + if args.vararg: + result.add(args.vararg.arg) + if args.kwarg: + result.add(args.kwarg.arg) + return result + + @cached_property + def dependencies(self) -> FrozenSet[str]: + """Names that are defined outside of the contract body. + + 1. Excludes built-in objects. + 1. Excludes contract function arguments. + """ + deps = set() + for node in ast.walk(self.body): if not isinstance(node, ast.Name): continue if hasattr(builtins, node.id): continue - # C-compiled built-in - if node.id in sys.builtin_module_names: - modules.add(node.id) + if node.id in self.arguments: continue - # stdlib - spec = find_spec(node.id) - if spec and spec.origin.startswith(stdlib_prefix): - modules.add(node.id) - continue - - # import inferred modules - for module in modules: - node: ast.Import = ast.parse('import something').body[0] - node.names[0].name = module - contract.body.insert(0, node) - - return contract + deps.add(node.id) + return frozenset(deps) @staticmethod def _resolve_name(contract): diff --git a/deal/linter/_func.py b/deal/linter/_func.py index c6337a44..4af261e2 100644 --- a/deal/linter/_func.py +++ b/deal/linter/_func.py @@ -1,7 +1,7 @@ # built-in import ast from pathlib import Path -from typing import Iterable, List, NamedTuple +from typing import Dict, Iterable, List, NamedTuple # external import astroid @@ -87,6 +87,34 @@ def from_astroid(cls, tree: astroid.Module) -> List['Func']: )) return funcs + @staticmethod + def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: + result: Dict[str, ast.AST] = dict() + for node in tree.body: + if isinstance(node, ast.Import): + for name_node in node.names: + stmt = ast.Import([name_node]) + name = name_node.asname or name_node.name + result[name] = stmt + continue + + if isinstance(node, ast.ImportFrom): + module_name = '.' * node.level + node.module + for name_node in node.names: + stmt = ast.ImportFrom(module_name, [name_node]) + name = name_node.asname or name_node.name + result[name] = stmt + continue + + if isinstance(node, ast.Expr): + node = node.value + if isinstance(node, ast.Assign): + for target in node.targets: + if not isinstance(target, ast.Name): + continue + result[target.id] = ast.Expr(node) + return result + def __repr__(self) -> str: return '{name}({cats})'.format( name=type(self).__name__, diff --git a/tests/test_linter/test_contract.py b/tests/test_linter/test_contract.py index f31aa7ae..37eead05 100644 --- a/tests/test_linter/test_contract.py +++ b/tests/test_linter/test_contract.py @@ -4,6 +4,7 @@ # external import astroid +import pytest # project from deal.linter._contract import Category, Contract @@ -135,3 +136,55 @@ def f(a, b): c = func.contracts[0] assert c.run(3, 2) is True assert c.run(2, 3) is False + + +@pytest.mark.parametrize('source, deps', [ + ('lambda: ...', set()), + ('lambda a, b: ...', {'a', 'b'}), + ('lambda *args, **kwargs: ...', {'args', 'kwargs'}), + ('lambda a, *, b: ...', {'a', 'b'}), +]) +def test_arguments(source: str, deps: set): + text = """ + import deal + + @deal.post({source}) + def f(): + return 2 + """ + text = text.format(source=source) + text = dedent(text).strip() + tree = ast.parse(text) + print(ast.dump(tree)) + funcs1 = Func.from_ast(tree) + assert len(funcs1) == 1 + func = funcs1[0] + assert len(func.contracts) == 1 + c = func.contracts[0] + assert c.arguments == deps + + +@pytest.mark.parametrize('source, deps', [ + ('lambda a, b: cd', {'cd'}), + ('lambda a, b: a+b', set()), + ('lambda a, b: (a+b)/c', {'c'}), + + ('lambda: re.compile()', {'re'}), + ('lambda a, b: ab.cd()', {'ab'}), +]) +def test_dependencies(source: str, deps: set): + text = """ + import deal + + @deal.post({source}) + def f(a, b): + return a + b + """ + text = text.format(source=source) + text = dedent(text).strip() + funcs1 = Func.from_ast(ast.parse(text)) + assert len(funcs1) == 1 + func = funcs1[0] + assert len(func.contracts) == 1 + c = func.contracts[0] + assert c.dependencies == deps diff --git a/tests/test_linter/test_func.py b/tests/test_linter/test_func.py index 5057935a..2769a407 100644 --- a/tests/test_linter/test_func.py +++ b/tests/test_linter/test_func.py @@ -3,6 +3,7 @@ # external import astroid +import pytest # project from deal.linter._func import Func @@ -53,3 +54,21 @@ def test_repr(): funcs2 = Func.from_astroid(astroid.parse(TEXT)) for func in (funcs1[0], funcs2[0]): assert repr(func) == 'Func(post, raises)' + + +@pytest.mark.parametrize('source, names', [ + ('import re', {'re'}), + ('import typing, types', {'typing', 'types'}), + ('import typing as types', {'types'}), + + ('from typing import List', {'List'}), + ('from typing import List, Dict', {'List', 'Dict'}), + + ('ab = 2', {'ab'}), + ('ab = cd = 23', {'ab', 'cd'}), +]) +def test_extract_defs_ast(source: str, names) -> None: + tree = ast.parse(source) + print(ast.dump(tree)) + defs = Func._extract_defs_ast(tree) + assert set(defs) == names From 32dc5c152da88cc4103feca28b738d3ced720b29 Mon Sep 17 00:00:00 2001 From: Gram Date: Mon, 21 Sep 2020 16:57:54 +0200 Subject: [PATCH 05/30] inject deps into contract --- deal/linter/_contract.py | 57 ++++++++++++++++++++++++++++++++++------ deal/linter/_func.py | 2 ++ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/deal/linter/_contract.py b/deal/linter/_contract.py index e9ac710f..bcb87231 100644 --- a/deal/linter/_contract.py +++ b/deal/linter/_contract.py @@ -2,6 +2,7 @@ import ast import builtins import enum +from copy import copy from pathlib import Path from typing import Dict, FrozenSet, Iterable @@ -126,7 +127,7 @@ def exceptions(self) -> list: return excs @cached_property - def bytecode(self): + def module(self) -> ast.Module: module = ast.parse(TEMPLATE) # inject function signature @@ -134,9 +135,18 @@ def bytecode(self): func.args = self.func_args module.body[3].value = func - # inject contract - contract = self.body + # collect definitions for contract external deps + deps = [] + for dep in self.dependencies: + definition = self.context.get(dep) + if not definition: + continue + deps.append(definition) + + # inject contract if contract is a function + contract = copy(self.body) if isinstance(contract, ast.FunctionDef): + contract.body = deps + contract.body # if contract is function, add it's definition and assign it's name # to `contract` variable. module.body = [contract] + module.body @@ -146,12 +156,43 @@ def bytecode(self): col_offset=1, ctx=ast.Load(), ) - else: - if isinstance(contract, ast.Expr): - contract = contract.value - module.body[2].value = contract + return module + + if isinstance(contract, ast.Expr): + contract = contract.value + + # Inject contract if contract is a lambda. + # We have to rebuild lambda into a function + # to inject dependencies inside the body. + if isinstance(contract, ast.Lambda): + body = list(deps) + return_node = ast.Return( + value=contract.body, + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) + body.append(return_node) + var_name = module.body[2].targets[0].id + func = ast.FunctionDef( + name=var_name, + args=contract.args, + body=body, + decorator_list=[], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) + module.body[2] = func + return module - return compile(module, filename='', mode='exec') + # inject contract if contract is an unknown expression + module.body[2].value = contract + return module + + @cached_property + def bytecode(self): + return compile(self.module, filename='', mode='exec') def run(self, *args, **kwargs): globals = dict(args=args, kwargs=kwargs) diff --git a/deal/linter/_func.py b/deal/linter/_func.py index 4af261e2..ad894ff8 100644 --- a/deal/linter/_func.py +++ b/deal/linter/_func.py @@ -34,6 +34,7 @@ def from_text(cls, text: str) -> List['Func']: @classmethod def from_ast(cls, tree: ast.Module) -> List['Func']: funcs = [] + definitions = cls._extract_defs_ast(tree=tree) for expr in tree.body: if not isinstance(expr, ast.FunctionDef): continue @@ -43,6 +44,7 @@ def from_ast(cls, tree: ast.Module) -> List['Func']: args=args, category=Category(category), func_args=expr.args, + context=definitions, ) contracts.append(contract) funcs.append(cls( From d2d01f19e220007ef0acbfd10ec95bb5915139c5 Mon Sep 17 00:00:00 2001 From: Gram Date: Mon, 21 Sep 2020 17:23:44 +0200 Subject: [PATCH 06/30] extract context from astroid --- deal/linter/_func.py | 36 ++++++++++++++++++++++++++++++++-- tests/test_linter/test_func.py | 7 ++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/deal/linter/_func.py b/deal/linter/_func.py index ad894ff8..420e61ec 100644 --- a/deal/linter/_func.py +++ b/deal/linter/_func.py @@ -60,6 +60,7 @@ def from_ast(cls, tree: ast.Module) -> List['Func']: @classmethod def from_astroid(cls, tree: astroid.Module) -> List['Func']: funcs = [] + definitions = cls._extract_defs_astroid(tree=tree) for expr in tree.body: if not isinstance(expr, astroid.FunctionDef): continue @@ -76,6 +77,7 @@ def from_astroid(cls, tree: astroid.Module) -> List['Func']: args=args, func_args=func_args, category=Category(category), + context=definitions, ) contracts.append(contract) @@ -95,7 +97,7 @@ def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: for node in tree.body: if isinstance(node, ast.Import): for name_node in node.names: - stmt = ast.Import([name_node]) + stmt = ast.Import(names=[name_node]) name = name_node.asname or name_node.name result[name] = stmt continue @@ -103,7 +105,7 @@ def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: if isinstance(node, ast.ImportFrom): module_name = '.' * node.level + node.module for name_node in node.names: - stmt = ast.ImportFrom(module_name, [name_node]) + stmt = ast.ImportFrom(module=module_name, names=[name_node]) name = name_node.asname or name_node.name result[name] = stmt continue @@ -117,6 +119,36 @@ def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: result[target.id] = ast.Expr(node) return result + @staticmethod + def _extract_defs_astroid(tree: astroid.Module) -> Dict[str, ast.AST]: + result: Dict[str, ast.AST] = dict() + for node in tree.body: + if isinstance(node, astroid.Import): + for name, alias in node.names: + result[alias or name] = ast.Import( + names=[ast.alias(name=name, asname=alias)], + ) + continue + + if isinstance(node, astroid.ImportFrom): + module_name = '.' * (node.level or 0) + node.modname + for name, alias in node.names: + result[alias or name] = ast.ImportFrom( + module=module_name, + names=[ast.alias(name=name, asname=alias)], + ) + continue + + if isinstance(node, astroid.Expr): + node = node.value + if isinstance(node, astroid.Assign): + expr = ast.parse(node.value.as_string()).body[0] + for target in node.targets: + if not isinstance(target, astroid.AssignName): + continue + result[target.name] = expr + return result + def __repr__(self) -> str: return '{name}({cats})'.format( name=type(self).__name__, diff --git a/tests/test_linter/test_func.py b/tests/test_linter/test_func.py index 2769a407..f1595c34 100644 --- a/tests/test_linter/test_func.py +++ b/tests/test_linter/test_func.py @@ -67,8 +67,13 @@ def test_repr(): ('ab = 2', {'ab'}), ('ab = cd = 23', {'ab', 'cd'}), ]) -def test_extract_defs_ast(source: str, names) -> None: +def test_extract_defs(source: str, names) -> None: tree = ast.parse(source) print(ast.dump(tree)) defs = Func._extract_defs_ast(tree) assert set(defs) == names + + tree = astroid.parse(source) + print(tree.repr_tree()) + defs = Func._extract_defs_astroid(tree) + assert set(defs) == names From 7704ae9e4fa83d91375efb1512abf1b3d7c1bc2a Mon Sep 17 00:00:00 2001 From: Gram Date: Mon, 21 Sep 2020 18:18:16 +0200 Subject: [PATCH 07/30] inject deps into astroid contracts --- deal/linter/_contract.py | 7 ++- deal/linter/_func.py | 25 +++++++-- tests/test_linter/test_contract.py | 81 +++++++++++++++++++++++++++--- tests/test_linter/test_func.py | 12 +++++ 4 files changed, 111 insertions(+), 14 deletions(-) diff --git a/deal/linter/_contract.py b/deal/linter/_contract.py index bcb87231..ae9e7f2d 100644 --- a/deal/linter/_contract.py +++ b/deal/linter/_contract.py @@ -65,9 +65,12 @@ def arguments(self) -> FrozenSet[str]: Useful for resolving external dependencies. """ - if not isinstance(self.body, (ast.FunctionDef, ast.Lambda)): + func = self.body + if isinstance(func, ast.Expr): + func = func.value + if not isinstance(func, (ast.FunctionDef, ast.Lambda)): return frozenset() - args = self.body.args + args = func.args result = set() for arg in args.args: result.add(arg.arg) diff --git a/deal/linter/_func.py b/deal/linter/_func.py index 420e61ec..eea2205c 100644 --- a/deal/linter/_func.py +++ b/deal/linter/_func.py @@ -97,7 +97,12 @@ def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: for node in tree.body: if isinstance(node, ast.Import): for name_node in node.names: - stmt = ast.Import(names=[name_node]) + stmt = ast.Import( + names=[name_node], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) name = name_node.asname or name_node.name result[name] = stmt continue @@ -105,7 +110,13 @@ def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: if isinstance(node, ast.ImportFrom): module_name = '.' * node.level + node.module for name_node in node.names: - stmt = ast.ImportFrom(module=module_name, names=[name_node]) + stmt = ast.ImportFrom( + module=module_name, + names=[name_node], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) name = name_node.asname or name_node.name result[name] = stmt continue @@ -116,7 +127,7 @@ def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: for target in node.targets: if not isinstance(target, ast.Name): continue - result[target.id] = ast.Expr(node) + result[target.id] = node return result @staticmethod @@ -127,6 +138,9 @@ def _extract_defs_astroid(tree: astroid.Module) -> Dict[str, ast.AST]: for name, alias in node.names: result[alias or name] = ast.Import( names=[ast.alias(name=name, asname=alias)], + lineno=1, + col_offset=1, + ctx=ast.Load(), ) continue @@ -136,13 +150,16 @@ def _extract_defs_astroid(tree: astroid.Module) -> Dict[str, ast.AST]: result[alias or name] = ast.ImportFrom( module=module_name, names=[ast.alias(name=name, asname=alias)], + lineno=1, + col_offset=1, + ctx=ast.Load(), ) continue if isinstance(node, astroid.Expr): node = node.value if isinstance(node, astroid.Assign): - expr = ast.parse(node.value.as_string()).body[0] + expr = ast.parse(node.as_string()).body[0] for target in node.targets: if not isinstance(target, astroid.AssignName): continue diff --git a/tests/test_linter/test_contract.py b/tests/test_linter/test_contract.py index 37eead05..41a3fa0b 100644 --- a/tests/test_linter/test_contract.py +++ b/tests/test_linter/test_contract.py @@ -154,14 +154,21 @@ def f(): """ text = text.format(source=source) text = dedent(text).strip() + tree = ast.parse(text) print(ast.dump(tree)) funcs1 = Func.from_ast(tree) - assert len(funcs1) == 1 - func = funcs1[0] - assert len(func.contracts) == 1 - c = func.contracts[0] - assert c.arguments == deps + + tree = astroid.parse(text) + print(tree.repr_tree()) + funcs2 = Func.from_astroid(tree) + + for funcs in (funcs1, funcs2): + assert len(funcs) == 1 + func = funcs[0] + assert len(func.contracts) == 1 + c = func.contracts[0] + assert c.arguments == deps @pytest.mark.parametrize('source, deps', [ @@ -183,8 +190,66 @@ def f(a, b): text = text.format(source=source) text = dedent(text).strip() funcs1 = Func.from_ast(ast.parse(text)) - assert len(funcs1) == 1 - func = funcs1[0] + + tree = astroid.parse(text) + print(tree.repr_tree()) + funcs2 = Func.from_astroid(tree) + + for funcs in (funcs1, funcs2): + assert len(funcs) == 1 + func = funcs[0] + assert len(func.contracts) == 1 + c = func.contracts[0] + assert c.dependencies == deps + + +def test_resolve_and_run_dependencies_func_astroid(): + text = """ + import deal + CONST = 34 + + def contract(a): + return a == CONST + + @deal.post(contract) + def f(a): + return a * 2 + """ + text = dedent(text).strip() + tree = astroid.parse(text) + print(tree.repr_tree()) + funcs = Func.from_astroid(tree) + assert len(funcs) == 2 + func = funcs[-1] assert len(func.contracts) == 1 c = func.contracts[0] - assert c.dependencies == deps + + c.run(12) is False + c.run(34) is True + + +def test_resolve_and_run_dependencies_lambda(): + text = """ + import deal + + CONST = 34 + + @deal.post(lambda a: a == CONST) + def f(a): + return a * 2 + """ + text = dedent(text).strip() + funcs1 = Func.from_ast(ast.parse(text)) + + tree = astroid.parse(text) + print(tree.repr_tree()) + funcs2 = Func.from_astroid(tree) + + for funcs in (funcs1, funcs2): + assert len(funcs) == 1 + func = funcs[0] + assert len(func.contracts) == 1 + c = func.contracts[0] + + c.run(12) is False + c.run(34) is True diff --git a/tests/test_linter/test_func.py b/tests/test_linter/test_func.py index f1595c34..0d85b0d7 100644 --- a/tests/test_linter/test_func.py +++ b/tests/test_linter/test_func.py @@ -73,7 +73,19 @@ def test_extract_defs(source: str, names) -> None: defs = Func._extract_defs_ast(tree) assert set(defs) == names + module = ast.parse('hello') + for name, stmt in defs.items(): + module.body[0] = stmt + print(name, '|>', ast.dump(module)) + compile(module, filename='', mode='exec') + tree = astroid.parse(source) print(tree.repr_tree()) defs = Func._extract_defs_astroid(tree) assert set(defs) == names + + module = ast.parse('hello') + for name, stmt in defs.items(): + module.body[0] = stmt + print(name, '|>', ast.dump(module)) + compile(module, filename='', mode='exec') From d3823415bc493fbd6a9559287b4ad96b91812501 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 10:25:10 +0200 Subject: [PATCH 08/30] mv defs extractor --- deal/linter/_extractors/__init__.py | 2 + deal/linter/_extractors/definitions.py | 87 +++++++++++++++++++ deal/linter/_func.py | 85 ++---------------- .../test_extractors/test_definitions.py | 39 +++++++++ tests/test_linter/test_func.py | 35 -------- 5 files changed, 133 insertions(+), 115 deletions(-) create mode 100644 deal/linter/_extractors/definitions.py create mode 100644 tests/test_linter/test_extractors/test_definitions.py diff --git a/deal/linter/_extractors/__init__.py b/deal/linter/_extractors/__init__.py index acac611d..24873200 100644 --- a/deal/linter/_extractors/__init__.py +++ b/deal/linter/_extractors/__init__.py @@ -2,6 +2,7 @@ from .asserts import get_asserts from .common import get_name from .contracts import get_contracts +from .definitions import get_definitions from .exceptions import get_exceptions from .imports import get_imports from .markers import get_markers @@ -13,6 +14,7 @@ __all__ = [ 'get_asserts', 'get_contracts', + 'get_definitions', 'get_exceptions', 'get_imports', 'get_markers', diff --git a/deal/linter/_extractors/definitions.py b/deal/linter/_extractors/definitions.py new file mode 100644 index 00000000..7d5cfcaa --- /dev/null +++ b/deal/linter/_extractors/definitions.py @@ -0,0 +1,87 @@ +import ast +import astroid +from typing import Dict, Union + + +TreeType = Union[ast.Module, astroid.Module] + + +def get_definitions(tree: TreeType) -> Dict[str, ast.AST]: + if isinstance(tree, ast.Module): + return _extract_defs_ast(tree) + return _extract_defs_astroid(tree) + + +def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: + result: Dict[str, ast.AST] = dict() + for node in tree.body: + if isinstance(node, ast.Import): + for name_node in node.names: + stmt = ast.Import( + names=[name_node], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) + name = name_node.asname or name_node.name + result[name] = stmt + continue + + if isinstance(node, ast.ImportFrom): + module_name = '.' * node.level + node.module + for name_node in node.names: + stmt = ast.ImportFrom( + module=module_name, + names=[name_node], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) + name = name_node.asname or name_node.name + result[name] = stmt + continue + + if isinstance(node, ast.Expr): + node = node.value + if isinstance(node, ast.Assign): + for target in node.targets: + if not isinstance(target, ast.Name): + continue + result[target.id] = node + return result + + +def _extract_defs_astroid(tree: astroid.Module) -> Dict[str, ast.AST]: + result: Dict[str, ast.AST] = dict() + for node in tree.body: + if isinstance(node, astroid.Import): + for name, alias in node.names: + result[alias or name] = ast.Import( + names=[ast.alias(name=name, asname=alias)], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) + continue + + if isinstance(node, astroid.ImportFrom): + module_name = '.' * (node.level or 0) + node.modname + for name, alias in node.names: + result[alias or name] = ast.ImportFrom( + module=module_name, + names=[ast.alias(name=name, asname=alias)], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) + continue + + if isinstance(node, astroid.Expr): + node = node.value + if isinstance(node, astroid.Assign): + expr = ast.parse(node.as_string()).body[0] + for target in node.targets: + if not isinstance(target, astroid.AssignName): + continue + result[target.name] = expr + return result diff --git a/deal/linter/_func.py b/deal/linter/_func.py index eea2205c..e9a32eaf 100644 --- a/deal/linter/_func.py +++ b/deal/linter/_func.py @@ -1,14 +1,14 @@ # built-in import ast from pathlib import Path -from typing import Dict, Iterable, List, NamedTuple +from typing import Iterable, List, NamedTuple # external import astroid # app from ._contract import Category, Contract -from ._extractors import get_contracts +from ._extractors import get_contracts, get_definitions class Func(NamedTuple): @@ -34,7 +34,7 @@ def from_text(cls, text: str) -> List['Func']: @classmethod def from_ast(cls, tree: ast.Module) -> List['Func']: funcs = [] - definitions = cls._extract_defs_ast(tree=tree) + definitions = get_definitions(tree=tree) for expr in tree.body: if not isinstance(expr, ast.FunctionDef): continue @@ -42,8 +42,8 @@ def from_ast(cls, tree: ast.Module) -> List['Func']: for category, args in get_contracts(expr.decorator_list): contract = Contract( args=args, - category=Category(category), func_args=expr.args, + category=Category(category), context=definitions, ) contracts.append(contract) @@ -60,7 +60,7 @@ def from_ast(cls, tree: ast.Module) -> List['Func']: @classmethod def from_astroid(cls, tree: astroid.Module) -> List['Func']: funcs = [] - definitions = cls._extract_defs_astroid(tree=tree) + definitions = get_definitions(tree=tree) for expr in tree.body: if not isinstance(expr, astroid.FunctionDef): continue @@ -91,81 +91,6 @@ def from_astroid(cls, tree: astroid.Module) -> List['Func']: )) return funcs - @staticmethod - def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: - result: Dict[str, ast.AST] = dict() - for node in tree.body: - if isinstance(node, ast.Import): - for name_node in node.names: - stmt = ast.Import( - names=[name_node], - lineno=1, - col_offset=1, - ctx=ast.Load(), - ) - name = name_node.asname or name_node.name - result[name] = stmt - continue - - if isinstance(node, ast.ImportFrom): - module_name = '.' * node.level + node.module - for name_node in node.names: - stmt = ast.ImportFrom( - module=module_name, - names=[name_node], - lineno=1, - col_offset=1, - ctx=ast.Load(), - ) - name = name_node.asname or name_node.name - result[name] = stmt - continue - - if isinstance(node, ast.Expr): - node = node.value - if isinstance(node, ast.Assign): - for target in node.targets: - if not isinstance(target, ast.Name): - continue - result[target.id] = node - return result - - @staticmethod - def _extract_defs_astroid(tree: astroid.Module) -> Dict[str, ast.AST]: - result: Dict[str, ast.AST] = dict() - for node in tree.body: - if isinstance(node, astroid.Import): - for name, alias in node.names: - result[alias or name] = ast.Import( - names=[ast.alias(name=name, asname=alias)], - lineno=1, - col_offset=1, - ctx=ast.Load(), - ) - continue - - if isinstance(node, astroid.ImportFrom): - module_name = '.' * (node.level or 0) + node.modname - for name, alias in node.names: - result[alias or name] = ast.ImportFrom( - module=module_name, - names=[ast.alias(name=name, asname=alias)], - lineno=1, - col_offset=1, - ctx=ast.Load(), - ) - continue - - if isinstance(node, astroid.Expr): - node = node.value - if isinstance(node, astroid.Assign): - expr = ast.parse(node.as_string()).body[0] - for target in node.targets: - if not isinstance(target, astroid.AssignName): - continue - result[target.name] = expr - return result - def __repr__(self) -> str: return '{name}({cats})'.format( name=type(self).__name__, diff --git a/tests/test_linter/test_extractors/test_definitions.py b/tests/test_linter/test_extractors/test_definitions.py new file mode 100644 index 00000000..7302df55 --- /dev/null +++ b/tests/test_linter/test_extractors/test_definitions.py @@ -0,0 +1,39 @@ +import ast +import astroid +import pytest +from deal.linter._extractors import get_definitions + + +@pytest.mark.parametrize('source, names', [ + ('import re', {'re'}), + ('import typing, types', {'typing', 'types'}), + ('import typing as types', {'types'}), + + ('from typing import List', {'List'}), + ('from typing import List, Dict', {'List', 'Dict'}), + + ('ab = 2', {'ab'}), + ('ab = cd = 23', {'ab', 'cd'}), +]) +def test_extract_defs(source: str, names) -> None: + tree = ast.parse(source) + print(ast.dump(tree)) + defs = get_definitions(tree) + assert set(defs) == names + + module = ast.parse('hello') + for name, stmt in defs.items(): + module.body[0] = stmt + print(name, '|>', ast.dump(module)) + compile(module, filename='', mode='exec') + + tree = astroid.parse(source) + print(tree.repr_tree()) + defs = get_definitions(tree) + assert set(defs) == names + + module = ast.parse('hello') + for name, stmt in defs.items(): + module.body[0] = stmt + print(name, '|>', ast.dump(module)) + compile(module, filename='', mode='exec') diff --git a/tests/test_linter/test_func.py b/tests/test_linter/test_func.py index 0d85b0d7..bc7102a9 100644 --- a/tests/test_linter/test_func.py +++ b/tests/test_linter/test_func.py @@ -54,38 +54,3 @@ def test_repr(): funcs2 = Func.from_astroid(astroid.parse(TEXT)) for func in (funcs1[0], funcs2[0]): assert repr(func) == 'Func(post, raises)' - - -@pytest.mark.parametrize('source, names', [ - ('import re', {'re'}), - ('import typing, types', {'typing', 'types'}), - ('import typing as types', {'types'}), - - ('from typing import List', {'List'}), - ('from typing import List, Dict', {'List', 'Dict'}), - - ('ab = 2', {'ab'}), - ('ab = cd = 23', {'ab', 'cd'}), -]) -def test_extract_defs(source: str, names) -> None: - tree = ast.parse(source) - print(ast.dump(tree)) - defs = Func._extract_defs_ast(tree) - assert set(defs) == names - - module = ast.parse('hello') - for name, stmt in defs.items(): - module.body[0] = stmt - print(name, '|>', ast.dump(module)) - compile(module, filename='', mode='exec') - - tree = astroid.parse(source) - print(tree.repr_tree()) - defs = Func._extract_defs_astroid(tree) - assert set(defs) == names - - module = ast.parse('hello') - for name, stmt in defs.items(): - module.body[0] = stmt - print(name, '|>', ast.dump(module)) - compile(module, filename='', mode='exec') From 2c07118dcefe296496335345e69c258f7b84ca91 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 11:29:04 +0200 Subject: [PATCH 09/30] support context for pre-conditions --- deal/linter/_extractors/pre.py | 9 +++++---- deal/linter/_extractors/returns.py | 1 - deal/linter/_rules.py | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/deal/linter/_extractors/pre.py b/deal/linter/_extractors/pre.py index ba96e38f..92be76e8 100644 --- a/deal/linter/_extractors/pre.py +++ b/deal/linter/_extractors/pre.py @@ -1,6 +1,6 @@ # built-in import ast -from typing import Iterator, Sequence +from typing import Any, Dict, Iterator # external import astroid @@ -15,7 +15,7 @@ @get_pre.register(astroid.Call) -def handle_call(expr: astroid.Call) -> Iterator[Token]: +def handle_call(expr: astroid.Call, context: Dict[str, ast.AST] = None) -> Iterator[Token]: # app from .._contract import Category, Contract @@ -26,7 +26,7 @@ def handle_call(expr: astroid.Call) -> Iterator[Token]: return args.append(value) - kwargs = {} + kwargs: Dict[str, Any] = {} for subnode in (expr.keywords or ()): value = get_value(expr=subnode.value) if value is UNKNOWN: @@ -49,6 +49,7 @@ def handle_call(expr: astroid.Call) -> Iterator[Token]: args=contract_args, category=Category.PRE, func_args=func_args, + context=context, ) try: result = contract.run(*args, **kwargs) @@ -63,7 +64,7 @@ def handle_call(expr: astroid.Call) -> Iterator[Token]: ) -def format_call_args(args: Sequence, kwargs: dict) -> str: +def format_call_args(args: list, kwargs: Dict[str, Any]) -> str: sep = ', ' args_s = sep.join(map(repr, args)) kwargs_s = sep.join(['{}={!r}'.format(k, v) for k, v in kwargs.items()]) diff --git a/deal/linter/_extractors/returns.py b/deal/linter/_extractors/returns.py index e0776cc9..072b9688 100644 --- a/deal/linter/_extractors/returns.py +++ b/deal/linter/_extractors/returns.py @@ -7,7 +7,6 @@ get_returns = Extractor() -inner_extractor = Extractor() def has_returns(body: list) -> bool: diff --git a/deal/linter/_rules.py b/deal/linter/_rules.py index 353f7a3b..885c859b 100644 --- a/deal/linter/_rules.py +++ b/deal/linter/_rules.py @@ -61,7 +61,8 @@ def __call__(self, func: Func, stubs: StubsManager = None) -> Iterator[Error]: # is a really expensive operation. if not func.contracts: return - for token in get_pre(body=func.body): + context = func.contracts[0].context + for token in get_pre(body=func.body, context=context): yield Error( code=self.code, text=token.marker or self.message, From 4701130d43ca5e54326e02251577fce7d413eb31 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 12:34:09 +0200 Subject: [PATCH 10/30] lazy import missed modules for contracts --- deal/linter/_contract.py | 13 +++--- deal/linter/_template.py | 46 +++++++++++++++++-- tests/test_linter/test_contract.py | 38 +++++++++++++++ .../test_extractors/test_common.py | 1 + tests/test_linter/test_func.py | 1 - 5 files changed, 88 insertions(+), 11 deletions(-) diff --git a/deal/linter/_contract.py b/deal/linter/_contract.py index ae9e7f2d..741e2365 100644 --- a/deal/linter/_contract.py +++ b/deal/linter/_contract.py @@ -11,6 +11,8 @@ TEMPLATE = (Path(__file__).parent / '_template.py').read_text() +CONTRACT_INDEX = 3 +FUNC_INDEX = 4 class Category(enum.Enum): @@ -136,7 +138,7 @@ def module(self) -> ast.Module: # inject function signature func = ast.parse('lambda:0').body[0].value func.args = self.func_args - module.body[3].value = func + module.body[FUNC_INDEX].value = func # collect definitions for contract external deps deps = [] @@ -153,7 +155,7 @@ def module(self) -> ast.Module: # if contract is function, add it's definition and assign it's name # to `contract` variable. module.body = [contract] + module.body - module.body[3].value = ast.Name( + module.body[FUNC_INDEX].value = ast.Name( id=contract.name, lineno=1, col_offset=1, @@ -176,9 +178,8 @@ def module(self) -> ast.Module: ctx=ast.Load(), ) body.append(return_node) - var_name = module.body[2].targets[0].id func = ast.FunctionDef( - name=var_name, + name='contract', args=contract.args, body=body, decorator_list=[], @@ -186,11 +187,11 @@ def module(self) -> ast.Module: col_offset=1, ctx=ast.Load(), ) - module.body[2] = func + module.body[CONTRACT_INDEX] = func return module # inject contract if contract is an unknown expression - module.body[2].value = contract + module.body[CONTRACT_INDEX].value = contract return module @cached_property diff --git a/deal/linter/_template.py b/deal/linter/_template.py index 13cdc3f7..31b15646 100644 --- a/deal/linter/_template.py +++ b/deal/linter/_template.py @@ -1,5 +1,7 @@ # This file is excluded from coverage. +from importlib import import_module + # project from deal import ContractError from deal._decorators.base import Base @@ -9,12 +11,48 @@ contract = ... func = ... -base = Base(validator=contract) # type: ignore -if func is not Ellipsis: - base.function = func + +def inject(name: str) -> bool: + """Import the given module and inject it into globals. + """ + try: + globals()[name] = import_module(name) + except ImportError: + return False + return True + + +def validate(*args, **kwargs) -> None: + """Run validator, trying to fix missed imports on the way. + """ + base = Base(validator=contract) # type: ignore + if func is not Ellipsis: + base.function = func + + old_name = None + for _ in range(10): # maximum 10 tries, just in case + try: + base.validate(*args, **kwargs) + return + except NameError as err: + # Oh no, we didn't properly inject a variable, + # and now it is missed. Let's try to import it + # as module and run the contract again. + + name = err.args[0].split("'")[1] + # the missed name haven't changed, injection didn't help + if name == old_name: + raise + # try to import missed module + ok = inject(name) + # the missed name is not as module, give up + if not ok: + raise + continue + try: - base.validate(*args, **kwargs) # type: ignore # noqa: F821 + validate(*args, **kwargs) # type: ignore # noqa: F821 except ContractError as exc: result = False if exc.args: diff --git a/tests/test_linter/test_contract.py b/tests/test_linter/test_contract.py index 41a3fa0b..6ca62022 100644 --- a/tests/test_linter/test_contract.py +++ b/tests/test_linter/test_contract.py @@ -253,3 +253,41 @@ def f(a): c.run(12) is False c.run(34) is True + + +def test_lazy_import_stdlib(): + text = """ + import deal + + @deal.post(lambda a: re.compile('^abc$').match(a)) + def f(a): + return a * 2 + """ + text = dedent(text).strip() + funcs = Func.from_ast(ast.parse(text)) + assert len(funcs) == 1 + func = funcs[0] + assert len(func.contracts) == 1 + c = func.contracts[0] + + c.run("bcd") is False + c.run("abc") is True + + +def test_unresolvable(): + text = """ + import deal + + @deal.post(lambda a: re.compile(unknown)) + def f(a): + return a * 2 + """ + text = dedent(text).strip() + funcs = Func.from_ast(ast.parse(text)) + assert len(funcs) == 1 + func = funcs[0] + assert len(func.contracts) == 1 + c = func.contracts[0] + + with pytest.raises(NameError): + c.run("bcd") diff --git a/tests/test_linter/test_extractors/test_common.py b/tests/test_linter/test_extractors/test_common.py index e05bb4ac..7fee0b0e 100644 --- a/tests/test_linter/test_extractors/test_common.py +++ b/tests/test_linter/test_extractors/test_common.py @@ -35,6 +35,7 @@ def test_get_name(text, expected): ('class C:\n def f(): pass\nC.f', [('', 'C.f')]), ('class C:\n def f(): pass\nc = C()\nc.f', [('', 'C.f')]), ('print', [('builtins', 'print')]), + ('"".format', [('builtins', 'str.format')]), ]) def test_infer(text, expected): tree = astroid.parse(text) diff --git a/tests/test_linter/test_func.py b/tests/test_linter/test_func.py index bc7102a9..5057935a 100644 --- a/tests/test_linter/test_func.py +++ b/tests/test_linter/test_func.py @@ -3,7 +3,6 @@ # external import astroid -import pytest # project from deal.linter._func import Func From d1f72f7f7cf55f534fd44033758adf4e22a045cc Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 12:53:00 +0200 Subject: [PATCH 11/30] use pygments to highlight the source code --- deal/_cli/_lint.py | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/deal/_cli/_lint.py b/deal/_cli/_lint.py index 73c3f6fa..5f36febb 100644 --- a/deal/_cli/_lint.py +++ b/deal/_cli/_lint.py @@ -10,6 +10,15 @@ from ..linter import Checker +try: + import pygments +except ImportError: + pygments = None +else: + from pygments.formatters import TerminalFormatter + from pygments.lexers import PythonLexer + + COLORS = dict( red='\033[91m', green='\033[92m', @@ -18,11 +27,30 @@ magenta='\033[95m', end='\033[0m', ) +NOCOLORS = dict( + red='', + green='', + yellow='', + blue='', + magenta='', + end='', +) TEMPLATE = ' {blue}{row}{end}:{blue}{col}{end} {magenta}{code}{end} {yellow}{text}{end}' VALUE = ' {magenta}({value}){end}' POINTER = '{magenta}^{end}' +def highlight(source: str) -> str: + if pygments is None: + return source + source = pygments.highlight( + code=source, + lexer=PythonLexer(), + formatter=TerminalFormatter(), + ) + return source.strip() + + def get_paths(path: Path) -> Iterator[Path]: """Recursively yields python files. """ @@ -64,6 +92,7 @@ def get_errors(paths: Iterable[Union[str, Path]]) -> Iterator[dict]: def get_parser() -> ArgumentParser: parser = ArgumentParser(prog='python3 -m deal lint') parser.add_argument('--json', action='store_true', help='json output') + parser.add_argument('--nocolor', action='store_true', help='colorless output') parser.add_argument('paths', nargs='*', default='.') return parser @@ -73,6 +102,9 @@ def lint_command(argv: Sequence[str]) -> int: args = parser.parse_args(argv) prev = None errors = list(get_errors(paths=args.paths)) + colors = COLORS + if args.nocolor: + colors = NOCOLORS for error in errors: if args.json: print(json.dumps(error)) @@ -80,18 +112,21 @@ def lint_command(argv: Sequence[str]) -> int: # print file path if error['path'] != prev: - print('{green}{path}{end}'.format(**COLORS, **error)) + print('{green}{path}{end}'.format(**colors, **error)) prev = error['path'] # print message - line = TEMPLATE.format(**COLORS, **error) + line = TEMPLATE.format(**colors, **error) if error['value']: - line += VALUE.format(**COLORS, **error) + line += VALUE.format(**colors, **error) print(line) # print code line - pointer = ' ' * error['col'] + POINTER.format(**COLORS) - content = error['content'] + '\n' + pointer + pointer = ' ' * error['col'] + POINTER.format(**colors) + content = error['content'] + if not args.nocolor: + content = highlight(content) + content += '\n' + pointer content = indent(dedent(content), prefix=' ') print(content) return len(errors) From f72a2848597d28ba6a7c6e40f0077180628c2887 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 13:48:34 +0200 Subject: [PATCH 12/30] decouple path finding --- deal/_cli/_common.py | 38 ++++++++++++++++++++++++++ deal/_cli/_lint.py | 50 +++++++++-------------------------- tests/test_cli/test_common.py | 25 ++++++++++++++++++ tests/test_cli/test_lint.py | 22 +-------------- 4 files changed, 77 insertions(+), 58 deletions(-) create mode 100644 deal/_cli/_common.py create mode 100644 tests/test_cli/test_common.py diff --git a/deal/_cli/_common.py b/deal/_cli/_common.py new file mode 100644 index 00000000..be0b73ec --- /dev/null +++ b/deal/_cli/_common.py @@ -0,0 +1,38 @@ +from pathlib import Path +from typing import Iterator + +try: + import pygments +except ImportError: + pygments = None +else: + from pygments.formatters import TerminalFormatter + from pygments.lexers import PythonLexer + + +def highlight(source: str) -> str: + if pygments is None: + return source + source = pygments.highlight( + code=source, + lexer=PythonLexer(), + formatter=TerminalFormatter(), + ) + return source.strip() + + +def get_paths(path: Path) -> Iterator[Path]: + """Recursively yields python files. + """ + if not path.exists(): + raise FileNotFoundError(str(path)) + if path.is_file(): + if path.suffix == '.py': + yield path + return + for subpath in path.iterdir(): + if subpath.name[0] == '.': + continue + if subpath.name == '__pycache__': + continue + yield from get_paths(subpath) diff --git a/deal/_cli/_lint.py b/deal/_cli/_lint.py index 5f36febb..a97cb631 100644 --- a/deal/_cli/_lint.py +++ b/deal/_cli/_lint.py @@ -8,15 +8,7 @@ # app from ..linter import Checker - - -try: - import pygments -except ImportError: - pygments = None -else: - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonLexer +from ._common import highlight, get_paths COLORS = dict( @@ -40,34 +32,6 @@ POINTER = '{magenta}^{end}' -def highlight(source: str) -> str: - if pygments is None: - return source - source = pygments.highlight( - code=source, - lexer=PythonLexer(), - formatter=TerminalFormatter(), - ) - return source.strip() - - -def get_paths(path: Path) -> Iterator[Path]: - """Recursively yields python files. - """ - if not path.exists(): - raise FileNotFoundError(str(path)) - if path.is_file(): - if path.suffix == '.py': - yield path - return - for subpath in path.iterdir(): - if subpath.name[0] == '.': - continue - if subpath.name == '__pycache__': - continue - yield from get_paths(subpath) - - def get_errors(paths: Iterable[Union[str, Path]]) -> Iterator[dict]: for arg in paths: for path in get_paths(Path(arg)): @@ -98,6 +62,18 @@ def get_parser() -> ArgumentParser: def lint_command(argv: Sequence[str]) -> int: + """Run linter against given files. + + ```python + python3 -m deal lint project/ + ``` + + Options: + + * `--json`: output violations as json per line (ndjson.org). + * `--nocolor`: output violations in human-friendly format but without colors. + Useful for running linter on CI. + """ parser = get_parser() args = parser.parse_args(argv) prev = None diff --git a/tests/test_cli/test_common.py b/tests/test_cli/test_common.py new file mode 100644 index 00000000..6f6b1864 --- /dev/null +++ b/tests/test_cli/test_common.py @@ -0,0 +1,25 @@ +# built-in +from pathlib import Path + +# external +import pytest + +# project +from deal._cli._lint import get_paths + + +def test_get_paths(tmp_path: Path): + (tmp_path / 'subdir').mkdir() + (tmp_path / 'subdir' / '__pycache__').mkdir() + (tmp_path / '.hidden').mkdir() + + (tmp_path / 'setup.py').touch() + (tmp_path / 'subdir' / 'ex.py').touch() + (tmp_path / '.hidden' / 'ex.py').touch() + (tmp_path / 'subdir' / '__pycache__' / 'ex.py').touch() + (tmp_path / 'setup.pl').touch() + actual = {p.relative_to(tmp_path) for p in get_paths(tmp_path)} + assert actual == {Path('setup.py'), Path('subdir/ex.py')} + + with pytest.raises(FileNotFoundError): + list(get_paths(tmp_path / 'not_exists')) diff --git a/tests/test_cli/test_lint.py b/tests/test_cli/test_lint.py index 3e6b1455..ebdf583e 100644 --- a/tests/test_cli/test_lint.py +++ b/tests/test_cli/test_lint.py @@ -1,11 +1,8 @@ # built-in from pathlib import Path -# external -import pytest - # project -from deal._cli._lint import get_errors, get_paths, lint_command +from deal._cli._lint import get_errors, lint_command TEXT = """ @@ -17,23 +14,6 @@ def f(x): """ -def test_get_paths(tmp_path: Path): - (tmp_path / 'subdir').mkdir() - (tmp_path / 'subdir' / '__pycache__').mkdir() - (tmp_path / '.hidden').mkdir() - - (tmp_path / 'setup.py').touch() - (tmp_path / 'subdir' / 'ex.py').touch() - (tmp_path / '.hidden' / 'ex.py').touch() - (tmp_path / 'subdir' / '__pycache__' / 'ex.py').touch() - (tmp_path / 'setup.pl').touch() - actual = {p.relative_to(tmp_path) for p in get_paths(tmp_path)} - assert actual == {Path('setup.py'), Path('subdir/ex.py')} - - with pytest.raises(FileNotFoundError): - list(get_paths(tmp_path / 'not_exists')) - - def test_get_errors(tmp_path: Path): (tmp_path / 'example.py').write_text(TEXT) errors = list(get_errors(paths=[tmp_path])) From eaea2df3c4548bb3a01e5179d62fabbb6b1024eb Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 13:54:09 +0200 Subject: [PATCH 13/30] use smart files discovery for all CLI commands --- deal/_cli/_stub.py | 9 +++++++-- deal/_cli/_test.py | 16 +++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/deal/_cli/_stub.py b/deal/_cli/_stub.py index c3d1d3d3..6723f774 100644 --- a/deal/_cli/_stub.py +++ b/deal/_cli/_stub.py @@ -1,10 +1,11 @@ # built-in from argparse import ArgumentParser from pathlib import Path -from typing import Sequence +from typing import List, Sequence # app from ..linter import StubsManager, generate_stub +from ._common import get_paths def stub_command(argv: Sequence[str]) -> int: @@ -13,7 +14,11 @@ def stub_command(argv: Sequence[str]) -> int: parser.add_argument('paths', nargs='+') args = parser.parse_args(argv) - paths = [Path(path) for path in args.paths] + paths: List[Path] = [] + for arg in args.paths: + for path in get_paths(Path(arg)): + paths.append(path) + roots = list(StubsManager.default_paths) + list(set(paths)) stubs = StubsManager(paths=roots) diff --git a/deal/_cli/_test.py b/deal/_cli/_test.py index 456e2f74..d4b0a6bf 100644 --- a/deal/_cli/_test.py +++ b/deal/_cli/_test.py @@ -13,6 +13,7 @@ from ..linter._contract import Category from ..linter._extractors.pre import format_call_args from ..linter._func import Func +from ._common import get_paths COLORS = dict( @@ -97,11 +98,12 @@ def test_command( args = parser.parse_args(argv) failed = 0 - for path in args.paths: - failed += run_tests( - path=Path(path), - root=root, - count=args.count, - stream=stream, - ) + for arg in args.paths: + for path in get_paths(Path(arg)): + failed += run_tests( + path=Path(path), + root=root, + count=args.count, + stream=stream, + ) return failed From 9670c0540d779f8d8088a7642c6d85f74321fc51 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 13:58:37 +0200 Subject: [PATCH 14/30] sort imports --- deal/_cli/_common.py | 2 ++ deal/_cli/_lint.py | 8 +++++--- deal/_imports.py | 2 +- deal/linter/_extractors/definitions.py | 5 ++++- deal/linter/_template.py | 1 + examples/printf.py | 2 ++ tests/test_linter/test_extractors/test_definitions.py | 5 +++++ 7 files changed, 20 insertions(+), 5 deletions(-) diff --git a/deal/_cli/_common.py b/deal/_cli/_common.py index be0b73ec..b5c909ac 100644 --- a/deal/_cli/_common.py +++ b/deal/_cli/_common.py @@ -1,6 +1,8 @@ +# built-in from pathlib import Path from typing import Iterator + try: import pygments except ImportError: diff --git a/deal/_cli/_lint.py b/deal/_cli/_lint.py index a97cb631..d78ad245 100644 --- a/deal/_cli/_lint.py +++ b/deal/_cli/_lint.py @@ -8,7 +8,7 @@ # app from ..linter import Checker -from ._common import highlight, get_paths +from ._common import get_paths, highlight COLORS = dict( @@ -62,7 +62,7 @@ def get_parser() -> ArgumentParser: def lint_command(argv: Sequence[str]) -> int: - """Run linter against given files. + """Run linter against the given files. ```python python3 -m deal lint project/ @@ -70,9 +70,11 @@ def lint_command(argv: Sequence[str]) -> int: Options: - * `--json`: output violations as json per line (ndjson.org). + * `--json`: output violations as [json per line](http://ndjson.org/). * `--nocolor`: output violations in human-friendly format but without colors. Useful for running linter on CI. + + Exit code is equal to the found violations count. """ parser = get_parser() args = parser.parse_args(argv) diff --git a/deal/_imports.py b/deal/_imports.py index b5a4774a..47e1072a 100644 --- a/deal/_imports.py +++ b/deal/_imports.py @@ -4,7 +4,7 @@ from types import ModuleType from typing import Any, Callable, List, Optional -# external +# project from _frozen_importlib_external import PathFinder # app diff --git a/deal/linter/_extractors/definitions.py b/deal/linter/_extractors/definitions.py index 7d5cfcaa..5586fff9 100644 --- a/deal/linter/_extractors/definitions.py +++ b/deal/linter/_extractors/definitions.py @@ -1,7 +1,10 @@ +# built-in import ast -import astroid from typing import Dict, Union +# external +import astroid + TreeType = Union[ast.Module, astroid.Module] diff --git a/deal/linter/_template.py b/deal/linter/_template.py index 31b15646..3d854c71 100644 --- a/deal/linter/_template.py +++ b/deal/linter/_template.py @@ -1,5 +1,6 @@ # This file is excluded from coverage. +# built-in from importlib import import_module # project diff --git a/examples/printf.py b/examples/printf.py index 99c5b224..5cb7bbbb 100644 --- a/examples/printf.py +++ b/examples/printf.py @@ -1,5 +1,7 @@ +# built-in import re +# project import deal diff --git a/tests/test_linter/test_extractors/test_definitions.py b/tests/test_linter/test_extractors/test_definitions.py index 7302df55..65703a0b 100644 --- a/tests/test_linter/test_extractors/test_definitions.py +++ b/tests/test_linter/test_extractors/test_definitions.py @@ -1,6 +1,11 @@ +# built-in import ast + +# external import astroid import pytest + +# project from deal.linter._extractors import get_definitions From a789964511df2f33696d77f720270cdd63aa5457 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 14:26:08 +0200 Subject: [PATCH 15/30] fix typing issues --- deal/linter/_contract.py | 35 ++++++++++++++++---------- deal/linter/_extractors/definitions.py | 28 ++++++++++----------- deal/linter/_extractors/pre.py | 6 ++--- deal/linter/_func.py | 4 +-- 4 files changed, 40 insertions(+), 33 deletions(-) diff --git a/deal/linter/_contract.py b/deal/linter/_contract.py index 741e2365..7d2afa9b 100644 --- a/deal/linter/_contract.py +++ b/deal/linter/_contract.py @@ -4,7 +4,7 @@ import enum from copy import copy from pathlib import Path -from typing import Dict, FrozenSet, Iterable +from typing import Dict, FrozenSet, Iterable, List # external import astroid @@ -38,14 +38,14 @@ class Contract: args: tuple category: Category func_args: ast.arguments - context: Dict[str, ast.AST] + context: Dict[str, ast.stmt] def __init__( self, args: Iterable, category: Category, func_args: ast.arguments, - context: Dict[str, ast.AST] = None, + context: Dict[str, ast.stmt] = None, ): self.args = tuple(args) self.category = category @@ -82,7 +82,7 @@ def arguments(self) -> FrozenSet[str]: result.add(args.vararg.arg) if args.kwarg: result.add(args.kwarg.arg) - return result + return frozenset(result) @cached_property def dependencies(self) -> FrozenSet[str]: @@ -136,12 +136,22 @@ def module(self) -> ast.Module: module = ast.parse(TEMPLATE) # inject function signature - func = ast.parse('lambda:0').body[0].value - func.args = self.func_args - module.body[FUNC_INDEX].value = func + func = ast.Lambda( + args=self.func_args, + body=ast.Set( + elts=[], + lineno=1, + col_offset=1, + ctx=ast.Load(), + ), + lineno=1, + col_offset=1, + ctx=ast.Load(), + ) + module.body[FUNC_INDEX].value = func # type: ignore # collect definitions for contract external deps - deps = [] + deps: List[ast.stmt] = [] for dep in self.dependencies: definition = self.context.get(dep) if not definition: @@ -154,8 +164,8 @@ def module(self) -> ast.Module: contract.body = deps + contract.body # if contract is function, add it's definition and assign it's name # to `contract` variable. - module.body = [contract] + module.body - module.body[FUNC_INDEX].value = ast.Name( + module.body = [contract] + module.body # type: ignore + module.body[FUNC_INDEX].value = ast.Name( # type: ignore id=contract.name, lineno=1, col_offset=1, @@ -178,7 +188,7 @@ def module(self) -> ast.Module: ctx=ast.Load(), ) body.append(return_node) - func = ast.FunctionDef( + module.body[CONTRACT_INDEX] = ast.FunctionDef( name='contract', args=contract.args, body=body, @@ -187,11 +197,10 @@ def module(self) -> ast.Module: col_offset=1, ctx=ast.Load(), ) - module.body[CONTRACT_INDEX] = func return module # inject contract if contract is an unknown expression - module.body[CONTRACT_INDEX].value = contract + module.body[CONTRACT_INDEX].value = contract # type: ignore return module @cached_property diff --git a/deal/linter/_extractors/definitions.py b/deal/linter/_extractors/definitions.py index 5586fff9..77471d53 100644 --- a/deal/linter/_extractors/definitions.py +++ b/deal/linter/_extractors/definitions.py @@ -7,45 +7,43 @@ TreeType = Union[ast.Module, astroid.Module] +DefsType = Dict[str, ast.stmt] -def get_definitions(tree: TreeType) -> Dict[str, ast.AST]: +def get_definitions(tree: TreeType) -> DefsType: if isinstance(tree, ast.Module): return _extract_defs_ast(tree) return _extract_defs_astroid(tree) -def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: - result: Dict[str, ast.AST] = dict() +def _extract_defs_ast(tree: ast.Module) -> DefsType: + result: DefsType = dict() for node in tree.body: if isinstance(node, ast.Import): for name_node in node.names: - stmt = ast.Import( + name = name_node.asname or name_node.name + result[name] = ast.Import( names=[name_node], lineno=1, col_offset=1, ctx=ast.Load(), ) - name = name_node.asname or name_node.name - result[name] = stmt continue if isinstance(node, ast.ImportFrom): - module_name = '.' * node.level + node.module + if not node.module or node.level: + continue for name_node in node.names: - stmt = ast.ImportFrom( - module=module_name, + name = name_node.asname or name_node.name + result[name] = ast.ImportFrom( + module=node.module, names=[name_node], lineno=1, col_offset=1, ctx=ast.Load(), ) - name = name_node.asname or name_node.name - result[name] = stmt continue - if isinstance(node, ast.Expr): - node = node.value if isinstance(node, ast.Assign): for target in node.targets: if not isinstance(target, ast.Name): @@ -54,8 +52,8 @@ def _extract_defs_ast(tree: ast.Module) -> Dict[str, ast.AST]: return result -def _extract_defs_astroid(tree: astroid.Module) -> Dict[str, ast.AST]: - result: Dict[str, ast.AST] = dict() +def _extract_defs_astroid(tree: astroid.Module) -> DefsType: + result: DefsType = dict() for node in tree.body: if isinstance(node, astroid.Import): for name, alias in node.names: diff --git a/deal/linter/_extractors/pre.py b/deal/linter/_extractors/pre.py index 92be76e8..07efb0cc 100644 --- a/deal/linter/_extractors/pre.py +++ b/deal/linter/_extractors/pre.py @@ -1,6 +1,6 @@ # built-in import ast -from typing import Any, Dict, Iterator +from typing import Any, Dict, Iterator, Sequence # external import astroid @@ -15,7 +15,7 @@ @get_pre.register(astroid.Call) -def handle_call(expr: astroid.Call, context: Dict[str, ast.AST] = None) -> Iterator[Token]: +def handle_call(expr: astroid.Call, context: Dict[str, ast.stmt] = None) -> Iterator[Token]: # app from .._contract import Category, Contract @@ -64,7 +64,7 @@ def handle_call(expr: astroid.Call, context: Dict[str, ast.AST] = None) -> Itera ) -def format_call_args(args: list, kwargs: Dict[str, Any]) -> str: +def format_call_args(args: Sequence, kwargs: Dict[str, Any]) -> str: sep = ', ' args_s = sep.join(map(repr, args)) kwargs_s = sep.join(['{}={!r}'.format(k, v) for k, v in kwargs.items()]) diff --git a/deal/linter/_func.py b/deal/linter/_func.py index e9a32eaf..f2967f2b 100644 --- a/deal/linter/_func.py +++ b/deal/linter/_func.py @@ -1,7 +1,7 @@ # built-in import ast from pathlib import Path -from typing import Iterable, List, NamedTuple +from typing import List, NamedTuple # external import astroid @@ -15,7 +15,7 @@ class Func(NamedTuple): name: str args: ast.arguments body: list - contracts: Iterable[Contract] + contracts: List[Contract] line: int col: int From 2947bf1293c5e80b26e6912afb2ff6b678740c8b Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 14:34:54 +0200 Subject: [PATCH 16/30] improve coverage --- deal/linter/_contract.py | 2 -- deal/linter/_extractors/definitions.py | 7 +++---- tests/test_linter/test_extractors/test_definitions.py | 3 +++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deal/linter/_contract.py b/deal/linter/_contract.py index 7d2afa9b..9f88498f 100644 --- a/deal/linter/_contract.py +++ b/deal/linter/_contract.py @@ -28,8 +28,6 @@ def __init__(self, func): self.func = func def __get__(self, obj, cls): - if obj is None: - return self value = obj.__dict__[self.func.__name__] = self.func(obj) return value diff --git a/deal/linter/_extractors/definitions.py b/deal/linter/_extractors/definitions.py index 77471d53..9b30620b 100644 --- a/deal/linter/_extractors/definitions.py +++ b/deal/linter/_extractors/definitions.py @@ -66,10 +66,11 @@ def _extract_defs_astroid(tree: astroid.Module) -> DefsType: continue if isinstance(node, astroid.ImportFrom): - module_name = '.' * (node.level or 0) + node.modname + if not node.modname or node.level: + continue for name, alias in node.names: result[alias or name] = ast.ImportFrom( - module=module_name, + module=node.modname, names=[ast.alias(name=name, asname=alias)], lineno=1, col_offset=1, @@ -77,8 +78,6 @@ def _extract_defs_astroid(tree: astroid.Module) -> DefsType: ) continue - if isinstance(node, astroid.Expr): - node = node.value if isinstance(node, astroid.Assign): expr = ast.parse(node.as_string()).body[0] for target in node.targets: diff --git a/tests/test_linter/test_extractors/test_definitions.py b/tests/test_linter/test_extractors/test_definitions.py index 65703a0b..2acac575 100644 --- a/tests/test_linter/test_extractors/test_definitions.py +++ b/tests/test_linter/test_extractors/test_definitions.py @@ -13,12 +13,15 @@ ('import re', {'re'}), ('import typing, types', {'typing', 'types'}), ('import typing as types', {'types'}), + ('from . import hi', set()), + ('from .something import hi', set()), ('from typing import List', {'List'}), ('from typing import List, Dict', {'List', 'Dict'}), ('ab = 2', {'ab'}), ('ab = cd = 23', {'ab', 'cd'}), + ('ab.cd = 2', set()), ]) def test_extract_defs(source: str, names) -> None: tree = ast.parse(source) From 0e2414df4415b81901c0be8ac88c9aa073c02869 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 22 Sep 2020 15:46:27 +0200 Subject: [PATCH 17/30] more docs --- README.md | 3 +++ docs/basic/tests.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 902b63e9..d07e3c35 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ * [External validators support.][validators] * [Contracts for importing modules.][module_load] * [Can be enabled or disabled on production.][runtime] +* [Colorless](colorless): annotate only what you want. Hence, easy integration into an existing project. +* Partial execution: linter executes contracts to statically check possible values. [values]: https://deal.readthedocs.io/basic/values.html [exceptions]: https://deal.readthedocs.io/basic/exceptions.html @@ -26,6 +28,7 @@ [validators]: https://deal.readthedocs.io/details/validators.html [module_load]: https://deal.readthedocs.io/details/module_load.html [runtime]: https://deal.readthedocs.io/basic/runtime.html +[colorless]: http://journal.stuffwithstuff.com/2015/02/01/what-color-is-your-function/ ## Deal in 30 seconds diff --git a/docs/basic/tests.md b/docs/basic/tests.md index cd397f73..ee30f72e 100644 --- a/docs/basic/tests.md +++ b/docs/basic/tests.md @@ -12,7 +12,7 @@ Then use `deal.cases` to get test cases for the function. Every case is a callab ```python @deal.raises(ZeroDivisionError) -@deal.pre(lambda a, b: a > 0 and b > 0) +@deal.pre(lambda a, b: a >= 0 and b >= 0) def div(a: int, b: int) -> float: return a / b From ebe8148ac9d5849d3811e2f2313dde5750515004 Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 13:52:21 +0200 Subject: [PATCH 18/30] drop old docs --- docs-old/commands/lint.md | 19 ---------- docs-old/decorators/chain.md | 19 ---------- docs-old/decorators/ensure.md | 68 ---------------------------------- docs-old/decorators/offline.md | 58 ----------------------------- docs-old/decorators/safe.md | 21 ----------- docs-old/decorators/silent.md | 50 ------------------------- docs-old/exceptions.md | 46 ----------------------- docs-old/index.md | 50 ------------------------- docs-old/validators.md | 62 ------------------------------- 9 files changed, 393 deletions(-) delete mode 100644 docs-old/commands/lint.md delete mode 100644 docs-old/decorators/chain.md delete mode 100644 docs-old/decorators/ensure.md delete mode 100644 docs-old/decorators/offline.md delete mode 100644 docs-old/decorators/safe.md delete mode 100644 docs-old/decorators/silent.md delete mode 100644 docs-old/exceptions.md delete mode 100644 docs-old/index.md delete mode 100644 docs-old/validators.md diff --git a/docs-old/commands/lint.md b/docs-old/commands/lint.md deleted file mode 100644 index 7b7a9c8d..00000000 --- a/docs-old/commands/lint.md +++ /dev/null @@ -1,19 +0,0 @@ -# **lint**: run static analysis - -Deal can do static checks for functions with contracts to catch trivial mistakes. Use [flake8](http://flake8.pycqa.org) or [flakehell](https://github.com/life4/flakehell) to run it. - -Another option is to use built-in CLI from deal: `python3 -m deal lint`. I has beautiful colored output by default. Use `--json` option to get compact JSON output. Pipe output into [jq](https://stedolan.github.io/jq/) to beautify JSON. - -![linter output](../../assets/linter.png) - -## Codes - -| Code | Message | -| ------- | ------------------------------------------------------------ | -| DEAL001 | do not use `from deal import ...`, use `import deal` instead | -| DEAL011 | pre contract error | -| DEAL012 | post contract error | -| DEAL021 | raises contract error | -| DEAL022 | silent contract error | -| DEAL023 | pure contract error | -| DEAL031 | assert error | diff --git a/docs-old/decorators/chain.md b/docs-old/decorators/chain.md deleted file mode 100644 index 4c5fa81c..00000000 --- a/docs-old/decorators/chain.md +++ /dev/null @@ -1,19 +0,0 @@ -# chain - -Beautiful way to apply a few short decorators to a function. - -```python -@deal.chain(deal.safe, deal.silent) -def show_division(a, b): - print(a / b) - -show_division(1, 2) -# SilentContractError: - -show_division(1, 0) -# RaisesContractError: -``` - -## Motivation - -Get rid of long chains of decorators because it looks like centipede. Use `@chain` to place short decorators horizontally. diff --git a/docs-old/decorators/ensure.md b/docs-old/decorators/ensure.md deleted file mode 100644 index bb63d666..00000000 --- a/docs-old/decorators/ensure.md +++ /dev/null @@ -1,68 +0,0 @@ -# ensure - -Ensure is a [postcondition](./post) that accepts not only result, but also function arguments. Must be true after function executed. Raises `PostContractError` otherwise. - -```python -@deal.ensure(lambda x, result: x != result) -def double(x): - return x * 2 - -double(2) -# 4 - -double(0) -# PostContractError: -``` - -For async functions it works the same. For generators validation runs for every yielded value: - -```python -@deal.ensure(lambda start, end, result: start <= result < end) -def range(start, end): - step = start - while step < end: - yield step - step += 1 -``` - -## Motivation - -Ensure allows you to simplify testing, easier check hypothesis, tell more about the function behavior. It works perfect for [P vs NP](https://en.wikipedia.org/wiki/P_versus_NP_problem) like problems. In other words, for complex task when checking result correctness (even partial checking only for some cases) is much easier then calculation itself. For example: - -```python -from typing import List - -# element at this position matches item -@deal.ensure( - lambda items, item, result: items[result] == item, - message='invalid match', -) -# element at this position is the first match -@deal.ensure( - lambda items, item, result: not any(el == item for el in items[:result]), - message='not the first match', -) -def index_of(items: List[int], item: int) -> int: - for index, element in enumerate(items): - if element == item: - return index - raise LookupError -``` - -Also, it's ok if you can check only some simple cases. For example, function `map` applies given function to the list. Let's check that count of returned elements is the same as the count of given elements: - -```python -from typing import Callable, List - -@deal.ensure(lambda: items, func, result: len(result) == len(items)) -def map(items: List[str], func: Callable[[str], str]) -> List[str]: - ... -``` - -Or if function `choice` returns random element from the list, we can't from one run check result randomness, but can't ensure that result is an element from the list: - -```python -@deal.ensure(lambda items, result: result in items) -def choice(items: List[str]) -> str: - ... -``` diff --git a/docs-old/decorators/offline.md b/docs-old/decorators/offline.md deleted file mode 100644 index 2d612a07..00000000 --- a/docs-old/decorators/offline.md +++ /dev/null @@ -1,58 +0,0 @@ -# offline - -Offline function cannot do network requests. This is achieved by patching `socket.socket`. So, don't use it into asynchronous or multi-threaded code. - -```python -@deal.offline -def ping(host): - if host == 'localhost': - return True - response = requests.head('http://' + host) - return response.ok - -ping('localhost') -# True - -ping('ya.ru') -# OfflineContractError: -``` - -It works the same for generators. For async functions keep in mind patching. - -## Motivation - -Sometimes, your code are doing unexpected network requests. Use `@offline` to catch these cases to do code optimization if possible. - -Bad: - -```python -def get_genres(): - response = requests.get('http://example.com/genres.txt') - response.raise_for_status() - return response.text.splitlines() - -def valid_genre(genre): - ... - return genre in get_genres() - -def validate_genres(genres): - return all(valid_genre(genre) for genre in genres) -``` - -Good: - -```python -def get_genres(): - response = requests.get('http://example.com/genres.txt') - response.raise_for_status() - return response.text.splitlines() - -@deal.offline -def valid_genre(genre, genres): - ... - return genre in genres - -def validate_genres(genres): - existing_genres = get_genres() - return all(valid_genre(genre, existing_genres) for genre in genres) -``` diff --git a/docs-old/decorators/safe.md b/docs-old/decorators/safe.md deleted file mode 100644 index 7fc81a5f..00000000 --- a/docs-old/decorators/safe.md +++ /dev/null @@ -1,21 +0,0 @@ -# safe - -Safe function cannot raise any exceptions. Alias for [raises](raises) without arguments. - -```python -@deal.safe -def divide(a, b): - return a / b - -divide(1, 2) -# 0.5 - -divide(1, 0) -# ZeroDivisionError: division by zero -# The above exception was the direct cause of the following exception: -# RaisesContractError: -``` - -## Motivation - -Can you be sure that some function never raises exceptions? Make promise about this, and `@deal.safe` will help you to control it. diff --git a/docs-old/decorators/silent.md b/docs-old/decorators/silent.md deleted file mode 100644 index d76b0f70..00000000 --- a/docs-old/decorators/silent.md +++ /dev/null @@ -1,50 +0,0 @@ -# silent - -Silent function cannot write anything into stdout. This is achieved by patching `sys.stdout`. So, don't use it into asynchronous or multi-threaded code. - -```python -@deal.silent -def has_access(role): - if role not in ('user', 'admin'): - print('role not found') - return False - return role == 'admin' - -has_access('admin') -# True - -has_access('superuser') -# SilentContractError: -``` - -It works the same for generators. For async functions keep in mind patching. - -## Motivation - -If possible, avoid any output from function. Direct output makes debugging and re-usage much more difficult. Of course, there are some exceptions: - -1. Entry-points that communicate with user and never should be used from other code. -1. Logging. Logging also output, but this output makes our life easier. However, be sure, you're not using logging for something important that should be checked from other code or tested. - -Bad: - -```python -def say_hello(name): - print('Hello, {name}') - -def main(): - say_hello(sys.argv[1]) -``` - -Good: - -```python -@deal.silent -def make_hello(name): - return 'Hello, {name}' - -def main(argv=None): - if argv is None: - argv = sys.argv[1:] - print(make_hello(argv[0])) -``` diff --git a/docs-old/exceptions.md b/docs-old/exceptions.md deleted file mode 100644 index 0ddcd94f..00000000 --- a/docs-old/exceptions.md +++ /dev/null @@ -1,46 +0,0 @@ -## Exceptions - -Every contract type has it's own exception type. Every exception inherited from `ContractError`. `ContractError` inherited from built-in `AssertionError`. - -Custom error message for any contract can be specified by `message` argument: - -```python -@deal.pre(lambda name: name.lower() != 'oleg', message='user name cannot be Oleg') -def hello(name): - print(f'hello, {name}') - -hello('Oleg') -# PreContractError: user name cannot be Oleg -``` - -Custom exception for any contract can be specified by `exception` argument: - -```python -@deal.pre(lambda role: role in ('user', 'admin'), exception=LookupError) -def change_role(role): - print(f'now you are {role}!') - -change_role('superuser') -# LookupError: -``` - -Also, contract can return string, and this string will be used as error message: - -```python -def contract(name): - if name.lower() == 'oleg': - return 'not today, Oleg' - if name in ('admin', 'moderator'): - return 'this name is reservered' - return True - -@deal.pre(contract) -def register(name): - print(f'welcome on board, {name}!') - -register('Greg') -# welcome on board, Greg! - -register('Oleg') -# PreContractError: not today, Oleg -``` diff --git a/docs-old/index.md b/docs-old/index.md deleted file mode 100644 index 7734874a..00000000 --- a/docs-old/index.md +++ /dev/null @@ -1,50 +0,0 @@ -```eval_rst -.. mdinclude:: ../README.md - -.. toctree:: - :maxdepth: 1 - :caption: Classic contracts - - decorators/pre - decorators/post - decorators/ensure - decorators/inv - -.. toctree:: - :maxdepth: 1 - :caption: CLI - - commands/lint - commands/stub - commands/test - -.. toctree:: - :maxdepth: 1 - :caption: Main info - - exceptions - disable - chaining - validators - testing - recipes - - -.. toctree:: - :maxdepth: 1 - :caption: Take more control - - decorators/module_load - decorators/offline - decorators/raises - decorators/reason - decorators/silent - -.. toctree:: - :maxdepth: 1 - :caption: Helpers - - decorators/chain - decorators/pure - decorators/safe -``` diff --git a/docs-old/validators.md b/docs-old/validators.md deleted file mode 100644 index 643ce633..00000000 --- a/docs-old/validators.md +++ /dev/null @@ -1,62 +0,0 @@ -# Validators - -## Simplified signature - -The main problem with contracts is that they have to duplicate the original function's signature, including default arguments. While it's not a problem for small examples, things become more complicated when the signature grows. For this case, you can specify a function that accepts only one `_` argument, and deal will pass here a container with arguments of the function call, including default ones: - -```python -@deal.pre(lambda _: _.a + _.b > 0) -def f(a, b=1): - return a + b - -f(1) -# 2 - -f(-2) -# PreContractError: -``` - -## Providing an error - -Regular contract can return error message instead of `False`: - -```python -def contract(name): - if not isinstance(name, str): - return 'name must be str' - return True - -@deal.pre(contract) -def f(x): - return x * 2 - -f('Chris') -# 'ChrisChris' - -f(4) -# PreContractError: name must be str -``` - -## External validators - -Deal supports a lot of external validation libraries, like Marshmallow, WTForms, PyScheme etc. For example: - -```python -import deal -import marshmallow - -class Schema(marshmallow.Schema): - name = marshmallow.fields.Str() - -@deal.pre(Schema) -def func(name): - return name * 2 - -func('Chris') -'ChrisChris' - -func(123) -# PreContractError: {'name': ['Not a valid string.']} -``` - -See [vaa](https://github.com/life4/vaa) documentation for details. From d0a29c3858b8342d57ee34fc81d17d3dc4565c4d Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 14:25:12 +0200 Subject: [PATCH 19/30] document CLI --- deal/_cli/_lint.py | 12 ++++++++---- deal/_cli/_stub.py | 16 ++++++++++++++++ deal/_cli/_test.py | 20 ++++++++++++++++++++ docs/details/cli.md | 19 +++++++++++++++++++ docs/index.md | 1 + 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 docs/details/cli.md diff --git a/deal/_cli/_lint.py b/deal/_cli/_lint.py index d78ad245..3c3a32c5 100644 --- a/deal/_cli/_lint.py +++ b/deal/_cli/_lint.py @@ -64,17 +64,21 @@ def get_parser() -> ArgumentParser: def lint_command(argv: Sequence[str]) -> int: """Run linter against the given files. - ```python + ```bash python3 -m deal lint project/ ``` Options: - * `--json`: output violations as [json per line](http://ndjson.org/). - * `--nocolor`: output violations in human-friendly format but without colors. - Useful for running linter on CI. + + `--json`: output violations as [json per line][ndjson]. + + `--nocolor`: output violations in human-friendly format but without colors. + Useful for running linter on CI. Exit code is equal to the found violations count. + See [linter][linter] documentation for more details. + + [ndjson]: http://ndjson.org/ + [linter]: https://deal.readthedocs.io/basic/linter.html """ parser = get_parser() args = parser.parse_args(argv) diff --git a/deal/_cli/_stub.py b/deal/_cli/_stub.py index 6723f774..1690fb46 100644 --- a/deal/_cli/_stub.py +++ b/deal/_cli/_stub.py @@ -9,6 +9,22 @@ def stub_command(argv: Sequence[str]) -> int: + """Generate stub files for the given Python files. + + ```bash + python3 -m deal stub project/ + ``` + + Options: + + + `--iterations`: how many time run stub generation against files. + Every new iteration uses results from the previous ones, improving the result. + Default: 1. + + Exit code is 0. See [stubs][stubs] documentation for more details. + + [stubs]: https://deal.readthedocs.io/details/stubs.html + """ parser = ArgumentParser(prog='python3 -m deal stub') parser.add_argument('--iterations', type=int, default=1) parser.add_argument('paths', nargs='+') diff --git a/deal/_cli/_test.py b/deal/_cli/_test.py index d4b0a6bf..3fc6f42d 100644 --- a/deal/_cli/_test.py +++ b/deal/_cli/_test.py @@ -90,6 +90,26 @@ def run_tests(path: Path, root: Path, count: int, stream: TextIO = sys.stdout) - def test_command( argv: Sequence[str], root: Path = None, stream: TextIO = sys.stdout, ) -> int: + """Generate and run tests against pure functions. + + ```bash + python3 -m deal test project/ + ``` + + Function must be decorated by one of the following to be run: + + + `@deal.pure` + + `@deal.has()` (without arguments) + + Options: + + + `--count`: how many input values combinations should be checked. + + Exit code is equal to count of failed test cases. + See [tests][tests] documentation for more details. + + [tests]: https://deal.readthedocs.io/basic/tests.html + """ if root is None: # pragma: no cover root = Path() parser = ArgumentParser(prog='python3 -m deal test') diff --git a/docs/details/cli.md b/docs/details/cli.md new file mode 100644 index 00000000..4529aafc --- /dev/null +++ b/docs/details/cli.md @@ -0,0 +1,19 @@ +# CLI + +## lint + +```eval_rst +.. autofunction:: deal._cli._lint.lint_command +``` + +## stub + +```eval_rst +.. autofunction:: deal._cli._stub.stub_command +``` + +## test + +```eval_rst +.. autofunction:: deal._cli._test.test_command +``` diff --git a/docs/index.md b/docs/index.md index a23a9fe9..9ac946d7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,5 +34,6 @@ details/stubs details/validators details/recipes + details/cli details/api ``` From 7d34a1f204f19d6cc34dacde6a7cce945d65c228 Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 14:28:47 +0200 Subject: [PATCH 20/30] make links in docstrings absolute --- deal/_aliases.py | 12 ++++++------ deal/_imports.py | 8 ++++++-- deal/_state.py | 4 +++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/deal/_aliases.py b/deal/_aliases.py index abc82a50..dc3b7cb6 100644 --- a/deal/_aliases.py +++ b/deal/_aliases.py @@ -47,7 +47,7 @@ def pre( ``` [wiki]: https://en.wikipedia.org/wiki/Precondition - [value]: ../basic/values.md + [value]: https://deal.readthedocs.io/basic/values.html """ cls = _decorators.Pre[_CallableType] return cls(validator=validator, message=message, exception=exception) @@ -90,7 +90,7 @@ def post( ``` [wiki]: https://en.wikipedia.org/wiki/Postcondition - [value]: ../basic/values.md + [value]: https://deal.readthedocs.io/basic/values.html """ cls = _decorators.Post[_CallableType] return cls(validator=validator, message=message, exception=exception) @@ -136,7 +136,7 @@ def ensure( ``` [wiki]: https://en.wikipedia.org/wiki/Postcondition - [value]: ../basic/values.md + [value]: https://deal.readthedocs.io/basic/values.html """ cls = _decorators.Ensure[_CallableType] return cls(validator=validator, message=message, exception=exception) @@ -184,7 +184,7 @@ def raises( ``` - [exception] ../basic/exceptions.md + [exception] https://deal.readthedocs.io/basic/exceptions.html """ cls = _decorators.Raises[_CallableType] return cls(*exceptions, message=message, exception=exception) @@ -245,7 +245,7 @@ def reason( ``` - [exception]: ../basic/exceptions.md + [exception]: https://deal.readthedocs.io/basic/exceptions.html """ cls = _decorators.Reason[_CallableType] return cls(event=event, validator=validator, message=message, exception=exception) @@ -308,7 +308,7 @@ def inv( ``` [wiki]: https://en.wikipedia.org/wiki/Class_invariant - [value]: ../basic/values.md + [value]: https://deal.readthedocs.io/basic/values.html """ cls = _decorators.Invariant[_CallableType] return cls( # type: ignore diff --git a/deal/_imports.py b/deal/_imports.py index 47e1072a..90c3921e 100644 --- a/deal/_imports.py +++ b/deal/_imports.py @@ -105,8 +105,10 @@ def module_load(*contracts) -> None: ``` - See [Contracts for importing modules](./module_load.md) + See [Contracts for importing modules][module_load] documentation for more details. + + [module_load]: https://deal.readthedocs.io/details/module_load.html """ if not state.debug: return @@ -133,8 +135,10 @@ def activate() -> bool: ``` - See [Contracts for importing modules](./module_load.md) + See [Contracts for importing modules][module_load] documentation for more details. + + [module_load]: https://deal.readthedocs.io/details/module_load.html """ if not state.debug: return False diff --git a/deal/_state.py b/deal/_state.py index 83a26ed2..3f4d526d 100644 --- a/deal/_state.py +++ b/deal/_state.py @@ -11,7 +11,9 @@ def reset(self) -> None: """Restore contracts switch to default. All contracts are disabled on production by default. - See [runtime](../basic/runtime.md) documentation. + See [runtime][runtime] documentation. + + [runtime]: https://deal.readthedocs.io/basic/runtime.html """ self.debug = __debug__ From 1d112910d9d2cb9375ec0649ca3364d94637de33 Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 14:54:32 +0200 Subject: [PATCH 21/30] include examples --- deal/_cli/_common.py | 2 +- docs/details/examples.md | 53 +++++++++++++++++++++++++++++++ docs/index.md | 1 + examples/choice.py | 4 +++ examples/concat.py | 5 +++ examples/count.py | 11 +++++++ examples/{printf.py => format.py} | 0 7 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/details/examples.md rename examples/{printf.py => format.py} (100%) diff --git a/deal/_cli/_common.py b/deal/_cli/_common.py index b5c909ac..bbef2afb 100644 --- a/deal/_cli/_common.py +++ b/deal/_cli/_common.py @@ -20,7 +20,7 @@ def highlight(source: str) -> str: lexer=PythonLexer(), formatter=TerminalFormatter(), ) - return source.strip() + return source.rstrip() def get_paths(path: Path) -> Iterator[Path]: diff --git a/docs/details/examples.md b/docs/details/examples.md new file mode 100644 index 00000000..92f5f256 --- /dev/null +++ b/docs/details/examples.md @@ -0,0 +1,53 @@ +# Examples + +## choice + +```eval_rst +.. literalinclude:: ../../examples/choice.py +``` + +## concat + +```eval_rst +.. literalinclude:: ../../examples/concat.py +``` + +## count + +```eval_rst +.. literalinclude:: ../../examples/count.py +``` + +## div + +```eval_rst +.. literalinclude:: ../../examples/div.py +``` + +## index_of + +```eval_rst +.. literalinclude:: ../../examples/index_of.py +``` + +## format + +```eval_rst +.. literalinclude:: ../../examples/format.py +``` + +Linter output: + +```bash +$ python3 -m deal lint examples/format.py +examples/format.py + 32:10 DEAL011 expected 1 argument(s) but 0 found ('{:s}') + print(format('{:s}')) # not enough args + ^ + 33:10 DEAL011 expected 1 argument(s) but 2 found ('{:s}', 'a', 'b') + print(format('{:s}', 'a', 'b')) # too many args + ^ + 34:10 DEAL011 expected float, str given ('{:d}', 'a') + print(format('{:d}', 'a')) # bad type + ^ +``` diff --git a/docs/index.md b/docs/index.md index 9ac946d7..75aff6d1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,6 +34,7 @@ details/stubs details/validators details/recipes + details/examples details/cli details/api ``` diff --git a/examples/choice.py b/examples/choice.py index 24891982..42d0ac16 100644 --- a/examples/choice.py +++ b/examples/choice.py @@ -9,10 +9,14 @@ import deal +# the list cannot be empty @deal.pre(lambda items: bool(items)) +# result is an element withit the given list @deal.ensure(lambda items, result: result in items) @deal.has() def choice(items: List[str]) -> str: + """Get a random element from the given list. + """ return random.choice(items) diff --git a/examples/concat.py b/examples/concat.py index 309af515..f22e7a1c 100644 --- a/examples/concat.py +++ b/examples/concat.py @@ -7,8 +7,13 @@ @deal.ensure(lambda _: _.result.startswith(_.left)) @deal.ensure(lambda _: _.result.endswith(_.right)) +@deal.ensure(lambda _: len(_.result) == len(_.left) + len(_.right)) @deal.has() def concat(left: str, right: str) -> str: + """Concatenate 2 given strings. + + https://deal.readthedocs.io/basic/motivation.html + """ return left + right diff --git a/examples/count.py b/examples/count.py index 59d83673..08d2f764 100644 --- a/examples/count.py +++ b/examples/count.py @@ -8,9 +8,20 @@ import deal +# In short signature, `_` is a `dict` with access by attributes. +# Hence it has all dict attributes. So, if argument we need conflicts +# with a dict attribute, use getitem instead of getattr. +# In the example below, we use `_['items']` instead of `_.items`. + @deal.post(lambda result: result >= 0) +# if count is not zero, `item` appears in `items` at least once. +@deal.ensure(lambda _: _.result == 0 or _['item'] in _['items']) +# if count is zero, `item` is not in `items` +@deal.ensure(lambda _: _.result != 0 or _['item'] not in _['items']) @deal.has() def count(items: List[str], item: str) -> int: + """How many times `item` appears in `items` + """ return items.count(item) diff --git a/examples/printf.py b/examples/format.py similarity index 100% rename from examples/printf.py rename to examples/format.py From 05338a36223ab9f5692188085355a1b3076df16c Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 16:59:15 +0200 Subject: [PATCH 22/30] document partial execution --- docs/basic/linter.md | 29 +++++++++++++++++++++++++++++ docs/details/recipes.md | 8 ++++++++ 2 files changed, 37 insertions(+) diff --git a/docs/basic/linter.md b/docs/basic/linter.md index fd4e21ce..70e8da94 100644 --- a/docs/basic/linter.md +++ b/docs/basic/linter.md @@ -79,3 +79,32 @@ Markers: | DEAL048 | missed marker (network) | +---------+-------------------------+ ``` + +## Partial execution + +To check `pre` and `post` contracts, linter can partially execute them. For example: + +```python +import deal + +@deal.post(lambda r: r != 0) +def f(): + return 0 +``` + +Try to run linter against the code above: + +```bash +$ python3 -m deal lint tmp.py +tmp.py + 6:11 DEAL012 post contract error (0) + return 0 +``` + +Hence there are some rules to make your contracts linter-friendly: + ++ Avoid side-effects, even logging. ++ Avoid external dependencies (functions and contants defined outside of the contract). ++ Keep them as small as possible. If you have a few different things to check, make separate contracts. + +Linter silently ignores contract if it cannot be executed. diff --git a/docs/details/recipes.md b/docs/details/recipes.md index e6e18ee4..52680aea 100644 --- a/docs/details/recipes.md +++ b/docs/details/recipes.md @@ -70,3 +70,11 @@ deal.module_load(deal.pure) ## Contracts shouldn't be important Never catch contract errors. Never rely on them in runtime. They are for tests and humans. The shouldn't have an actual logic, only validate it. + +## Short signature conflicts + +In short signature, `_` is a `dict` with access by attributes. Hence it has all dict attributes. So, if argument we need conflicts with a dict attribute, use getitem instead of getattr. For example, we should use `_['items']` instead of `_.items`. + +## What can be contract + +You can use any logic inside the validator. However, thumb up rule is to keep contracts [pure](https://en.wikipedia.org/wiki/Pure_function) (without any side-effects, even logging). The main motivation for it is that some contracts can be partially executed by [linter](../basic/linter.md). From cee92de6a44aa63a8decb613657a63b56e6a5068 Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 17:30:35 +0200 Subject: [PATCH 23/30] +min example --- docs/details/examples.md | 16 ++++++++++++++++ examples/index_of.py | 3 --- examples/min.py | 27 +++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 examples/min.py diff --git a/docs/details/examples.md b/docs/details/examples.md index 92f5f256..3d47ff2a 100644 --- a/docs/details/examples.md +++ b/docs/details/examples.md @@ -30,6 +30,22 @@ .. literalinclude:: ../../examples/index_of.py ``` +## min + +```eval_rst +.. literalinclude:: ../../examples/min.py +``` + +Linter output: + +```bash +$ python3 -m deal lint examples/min.py +examples/min.py + 21:4 DEAL011 pre contract error ([]) + my_min([]) + ^ +``` + ## format ```eval_rst diff --git a/examples/index_of.py b/examples/index_of.py index a2b8797e..186d79d6 100644 --- a/examples/index_of.py +++ b/examples/index_of.py @@ -41,9 +41,6 @@ def index_of(items: List[int], item: int) -> int: raise LookupError -# TESTS - - @pytest.mark.parametrize('case', deal.cases(index_of)) def test_index_of(case): case() diff --git a/examples/min.py b/examples/min.py new file mode 100644 index 00000000..a1fcbe8f --- /dev/null +++ b/examples/min.py @@ -0,0 +1,27 @@ +from typing import List, TypeVar + +import deal +import pytest + + +T = TypeVar('T') + + +@deal.pre(lambda items: len(items) > 0) +@deal.has() +def my_min(items: List[T]) -> T: + return min(items) + + +@deal.has() +def example(): + # good + my_min([3, 1, 4]) + # bad + my_min([]) + return 0 + + +@pytest.mark.parametrize('case', deal.cases(my_min)) +def test_min(case): + case() From 758b13fd14acf87924ba8d436121f49ff773cafd Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 17:37:58 +0200 Subject: [PATCH 24/30] test --nocolor --- tests/test_cli/test_lint.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_cli/test_lint.py b/tests/test_cli/test_lint.py index ebdf583e..ff54e580 100644 --- a/tests/test_cli/test_lint.py +++ b/tests/test_cli/test_lint.py @@ -33,6 +33,16 @@ def test_lint_command(tmp_path: Path, capsys): assert '^' in captured.out +def test_lint_command_no_color(tmp_path: Path, capsys): + (tmp_path / 'example.py').write_text(TEXT) + count = lint_command(['--nocolor', str(tmp_path)]) + assert count == 1 + + captured = capsys.readouterr() + exp = '6:11 DEAL012 post contract error (-1) return -1 ^' + assert captured.out.split()[1:] == exp.split() + + def test_lint_command_two_files(tmp_path: Path, capsys): (tmp_path / 'example1.py').write_text(TEXT) (tmp_path / 'example2.py').write_text(TEXT) From 5970201d708bd53a7ef68f9b8b41ac3384482252 Mon Sep 17 00:00:00 2001 From: Gram Date: Wed, 23 Sep 2020 17:52:15 +0200 Subject: [PATCH 25/30] color tracebacks --- deal/_cli/_common.py | 2 +- deal/_cli/_test.py | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/deal/_cli/_common.py b/deal/_cli/_common.py index bbef2afb..b5c909ac 100644 --- a/deal/_cli/_common.py +++ b/deal/_cli/_common.py @@ -20,7 +20,7 @@ def highlight(source: str) -> str: lexer=PythonLexer(), formatter=TerminalFormatter(), ) - return source.rstrip() + return source.strip() def get_paths(path: Path) -> Iterator[Path]: diff --git a/deal/_cli/_test.py b/deal/_cli/_test.py index 3fc6f42d..b32938aa 100644 --- a/deal/_cli/_test.py +++ b/deal/_cli/_test.py @@ -15,6 +15,14 @@ from ..linter._func import Func from ._common import get_paths +try: + import pygments +except ImportError: + pygments = None +else: + from pygments.formatters import TerminalFormatter + from pygments.lexers import PythonTracebackLexer + COLORS = dict( red='\033[91m', @@ -52,10 +60,21 @@ def get_func_names(path: Path) -> Iterator[str]: yield func.name +def color_exception(text: str) -> str: + text = text.replace('deal._exceptions.', '') + if pygments is None: + return '{red}{text}{end}'.format(text=text, **COLORS) + return pygments.highlight( + code=text, + lexer=PythonTracebackLexer(), + formatter=TerminalFormatter(), + ) + + def print_exception(stream: TextIO) -> None: lines = format_exception(*sys.exc_info()) - text = indent(text=''.join(lines), prefix=' ') - text = '{red}{text}{end}'.format(text=text, **COLORS) + text = color_exception(''.join(lines)) + text = indent(text=text, prefix=' ').rstrip() print(text, file=stream) From 6d0d218641d711a13fcf171aada77f084c85c4f5 Mon Sep 17 00:00:00 2001 From: Gram Date: Mon, 28 Sep 2020 15:48:01 +0200 Subject: [PATCH 26/30] better test TypeVar, test input args types --- deal/_cli/_common.py | 2 +- deal/_testing.py | 22 +++++++++++++++++++-- examples/min.py | 7 +++---- tests/test_testing.py | 46 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/deal/_cli/_common.py b/deal/_cli/_common.py index b5c909ac..bbef2afb 100644 --- a/deal/_cli/_common.py +++ b/deal/_cli/_common.py @@ -20,7 +20,7 @@ def highlight(source: str) -> str: lexer=PythonLexer(), formatter=TerminalFormatter(), ) - return source.strip() + return source.rstrip() def get_paths(path: Path) -> Iterator[Path]: diff --git a/deal/_testing.py b/deal/_testing.py index c4162fef..45776f6f 100644 --- a/deal/_testing.py +++ b/deal/_testing.py @@ -32,6 +32,10 @@ class TestCase(typing.NamedTuple): """Exceptions that must be suppressed. """ + check_types: bool + """Check that the result matches return type of the function. + """ + def __call__(self) -> typing.Any: """Calls the given test case returning the called functions result on success or Raising an exception on error @@ -44,13 +48,22 @@ def __call__(self) -> typing.Any: return result def _check_result(self, result: typing.Any) -> None: + if not self.check_types: + return hints = typing.get_type_hints(self.func) if 'return' not in hints: return + memo = typeguard._CallMemo( + func=self.func, + args=self.args, + kwargs=self.kwargs, + ) + typeguard.check_argument_types(memo=memo) typeguard.check_type( - argname='return', + argname='the return value', value=result, expected_type=hints['return'], + memo=memo, ) @@ -106,15 +119,19 @@ def example_generator(ex: ArgsKwargsType) -> None: def cases(func: typing.Callable, *, count: int = 50, kwargs: typing.Dict[str, typing.Any] = None, + check_types: bool = True, ) -> typing.Iterator[TestCase]: """[summary] :param func: the function to test. Should be type annotated. :type func: typing.Callable - :param count: how many test cases to generate, defaults to 50 + :param count: how many test cases to generate, defaults to 50. :type count: int, optional :param kwargs: keyword arguments to pass into the function. :type kwargs: typing.Dict[str, typing.Any], optional + :param check_types: check that the result matches return type of the function. + Enabled by default. + :type check_types: bool, optional :yield: Emits test cases. :rtype: typing.Iterator[TestCase] @@ -146,4 +163,5 @@ def cases(func: typing.Callable, *, count: int = 50, kwargs=kwargs, func=func, exceptions=tuple(get_excs(func)), + check_types=check_types, ) diff --git a/examples/min.py b/examples/min.py index a1fcbe8f..8c4d85c2 100644 --- a/examples/min.py +++ b/examples/min.py @@ -13,13 +13,12 @@ def my_min(items: List[T]) -> T: return min(items) -@deal.has() +@deal.has('stdout') def example(): # good - my_min([3, 1, 4]) + print(my_min([3, 1, 4])) # bad - my_min([]) - return 0 + print(my_min([])) @pytest.mark.parametrize('case', deal.cases(my_min)) diff --git a/tests/test_testing.py b/tests/test_testing.py index c6487f51..a04b3275 100644 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -1,5 +1,5 @@ # built-in -from typing import NoReturn +from typing import NoReturn, TypeVar # external import hypothesis @@ -82,3 +82,47 @@ def div(a: int, b: int): ) for case in cases: case() + + +def test_disable_type_checks(): + def bad(a: int) -> str: + return a + + # type is wrong and checked + cases = deal.cases(bad, count=1) + case = next(cases) + msg = 'type of the return value must be str; got int instead' + with pytest.raises(TypeError, match=msg): + case() + + # type is wrong and ignored + cases = deal.cases(bad, count=1, check_types=False) + case = next(cases) + case() + + def good(a: int) -> int: + return a + + # type is good + cases = deal.cases(good, count=1) + case = next(cases) + case() + + +def test_type_var(): + T = TypeVar('T') + + def identity(a: T, b) -> T: + return b + + kwargs = dict(kwargs={}, func=identity, exceptions=(), check_types=True) + case = deal.TestCase(args=('ab', 'cd'), **kwargs) + case() + + case = deal.TestCase(args=(['ab'], ['cd', 'ef']), **kwargs) + case() + + case = deal.TestCase(args=('ab', 1), **kwargs) + msg = 'type of the return value must be exactly str; got int instead' + with pytest.raises(TypeError, match=msg): + case() From 110bfaa46bcc886f73e72d69518f4772d5971a40 Mon Sep 17 00:00:00 2001 From: Gram Date: Mon, 28 Sep 2020 15:59:50 +0200 Subject: [PATCH 27/30] make pygments required --- deal/_cli/_common.py | 29 ++++++++++++++++++++--------- deal/_cli/_lint.py | 22 +++------------------- deal/_cli/_test.py | 26 +++++--------------------- deal/_testing.py | 17 +++++++++++------ pyproject.toml | 1 + tests/test_cli/test_lint.py | 12 ++++++------ 6 files changed, 46 insertions(+), 61 deletions(-) diff --git a/deal/_cli/_common.py b/deal/_cli/_common.py index bbef2afb..b173b682 100644 --- a/deal/_cli/_common.py +++ b/deal/_cli/_common.py @@ -2,19 +2,30 @@ from pathlib import Path from typing import Iterator +import pygments +from pygments.formatters import TerminalFormatter +from pygments.lexers import PythonLexer -try: - import pygments -except ImportError: - pygments = None -else: - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonLexer + +COLORS = dict( + red='\033[91m', + green='\033[92m', + yellow='\033[93m', + blue='\033[94m', + magenta='\033[95m', + end='\033[0m', +) +NOCOLORS = dict( + red='', + green='', + yellow='', + blue='', + magenta='', + end='', +) def highlight(source: str) -> str: - if pygments is None: - return source source = pygments.highlight( code=source, lexer=PythonLexer(), diff --git a/deal/_cli/_lint.py b/deal/_cli/_lint.py index 3c3a32c5..45200fbe 100644 --- a/deal/_cli/_lint.py +++ b/deal/_cli/_lint.py @@ -8,25 +8,9 @@ # app from ..linter import Checker -from ._common import get_paths, highlight - - -COLORS = dict( - red='\033[91m', - green='\033[92m', - yellow='\033[93m', - blue='\033[94m', - magenta='\033[95m', - end='\033[0m', -) -NOCOLORS = dict( - red='', - green='', - yellow='', - blue='', - magenta='', - end='', -) +from ._common import get_paths, highlight, COLORS, NOCOLORS + + TEMPLATE = ' {blue}{row}{end}:{blue}{col}{end} {magenta}{code}{end} {yellow}{text}{end}' VALUE = ' {magenta}({value}){end}' POINTER = '{magenta}^{end}' diff --git a/deal/_cli/_test.py b/deal/_cli/_test.py index b32938aa..b1c62d19 100644 --- a/deal/_cli/_test.py +++ b/deal/_cli/_test.py @@ -8,30 +8,16 @@ from traceback import format_exception from typing import Iterator, Sequence, TextIO +import pygments +from pygments.formatters import TerminalFormatter +from pygments.lexers import PythonTracebackLexer + # app from .._testing import cases from ..linter._contract import Category from ..linter._extractors.pre import format_call_args from ..linter._func import Func -from ._common import get_paths - -try: - import pygments -except ImportError: - pygments = None -else: - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonTracebackLexer - - -COLORS = dict( - red='\033[91m', - green='\033[92m', - yellow='\033[93m', - blue='\033[94m', - magenta='\033[95m', - end='\033[0m', -) +from ._common import get_paths, COLORS @contextmanager @@ -62,8 +48,6 @@ def get_func_names(path: Path) -> Iterator[str]: def color_exception(text: str) -> str: text = text.replace('deal._exceptions.', '') - if pygments is None: - return '{red}{text}{end}'.format(text=text, **COLORS) return pygments.highlight( code=text, lexer=PythonTracebackLexer(), diff --git a/deal/_testing.py b/deal/_testing.py index 45776f6f..020fd047 100644 --- a/deal/_testing.py +++ b/deal/_testing.py @@ -85,8 +85,11 @@ def get_excs(func: typing.Callable) -> typing.Iterator[typing.Type[Exception]]: func = func.__wrapped__ # type: ignore -def get_examples(func: typing.Callable, kwargs: typing.Dict[str, typing.Any], - count: int) -> typing.List[ArgsKwargsType]: +def get_examples( + func: typing.Callable, + kwargs: typing.Dict[str, typing.Any], + count: int, +) -> typing.List[ArgsKwargsType]: kwargs = kwargs.copy() for name, value in kwargs.items(): if isinstance(value, hypothesis.strategies.SearchStrategy): @@ -117,10 +120,12 @@ def example_generator(ex: ArgsKwargsType) -> None: return examples -def cases(func: typing.Callable, *, count: int = 50, - kwargs: typing.Dict[str, typing.Any] = None, - check_types: bool = True, - ) -> typing.Iterator[TestCase]: +def cases( + func: typing.Callable, *, + count: int = 50, + kwargs: typing.Dict[str, typing.Any] = None, + check_types: bool = True, +) -> typing.Iterator[TestCase]: """[summary] :param func: the function to test. Should be type annotated. diff --git a/pyproject.toml b/pyproject.toml index dd448639..8822eac1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ classifiers=[ python = ">=3.6" astroid = "*" hypothesis = "*" +pygments = "*" typeguard = "*" vaa = ">=0.2.1" diff --git a/tests/test_cli/test_lint.py b/tests/test_cli/test_lint.py index ff54e580..16fb5131 100644 --- a/tests/test_cli/test_lint.py +++ b/tests/test_cli/test_lint.py @@ -22,15 +22,15 @@ def test_get_errors(tmp_path: Path): assert errors[0]['content'] == ' return -1' -def test_lint_command(tmp_path: Path, capsys): +def test_lint_command_colors(tmp_path: Path, capsys): (tmp_path / 'example.py').write_text(TEXT) count = lint_command([str(tmp_path)]) assert count == 1 captured = capsys.readouterr() - assert 'return -1' in captured.out - assert '(-1)' in captured.out - assert '^' in captured.out + assert '\x1b[34mreturn\x1b[39;49;00m -\x1b[34m1\x1b[39;49;00m' in captured.out + assert '\x1b[95m(-1)\x1b[0m' in captured.out + assert '\x1b[95m^\x1b[0m' in captured.out def test_lint_command_no_color(tmp_path: Path, capsys): @@ -46,7 +46,7 @@ def test_lint_command_no_color(tmp_path: Path, capsys): def test_lint_command_two_files(tmp_path: Path, capsys): (tmp_path / 'example1.py').write_text(TEXT) (tmp_path / 'example2.py').write_text(TEXT) - count = lint_command([str(tmp_path)]) + count = lint_command(['--nocolor', str(tmp_path)]) assert count == 2 captured = capsys.readouterr() @@ -59,7 +59,7 @@ def test_lint_command_two_files(tmp_path: Path, capsys): def test_lint_command_two_errors(tmp_path: Path, capsys): (tmp_path / 'example.py').write_text('from deal import pre\n' + TEXT) - count = lint_command([str(tmp_path)]) + count = lint_command(['--nocolor', str(tmp_path)]) assert count == 2 captured = capsys.readouterr() From ec2c03347e6cc6e4321880bbdbd91aedcdcafcec Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 29 Sep 2020 13:40:24 +0200 Subject: [PATCH 28/30] less depend on astroid --- deal/_imports.py | 12 +++++++++++- docs/details/recipes.md | 6 +++++- tests/test_imports.py | 17 +++++++++++++++-- tests/test_main.py | 18 ++++++++++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/deal/_imports.py b/deal/_imports.py index 90c3921e..2ccc389b 100644 --- a/deal/_imports.py +++ b/deal/_imports.py @@ -10,7 +10,17 @@ # app from . import _aliases from ._state import state -from .linter._extractors.common import get_name + + +def get_name(expr) -> Optional[str]: + if isinstance(expr, ast.Name): + return expr.id + if isinstance(expr, ast.Attribute): + left = get_name(expr.value) + if left is None: + return None + return left + '.' + expr.attr + return None class DealFinder(PathFinder): diff --git a/docs/details/recipes.md b/docs/details/recipes.md index 52680aea..f5743e28 100644 --- a/docs/details/recipes.md +++ b/docs/details/recipes.md @@ -27,7 +27,7 @@ def div(left: float, right: float, default: float = None) -> float: ## Don't check types -Never check types with deal. [Mypy](https://github.com/python/mypy) does it much better. Also, there are [plenty of alternatives](https://github.com/typeddjango/awesome-python-typing) for both static and dynamic validation. Deal is intended to empower types, to tell a bit more about possible values set than you can do with type annotations, not replace them. However, if you want to play with deal a bit or make types a part of contracts, [PySchemes](https://github.com/spy16/pyschemes)-based contract is the best choice: +Never check types with deal. [MyPy](https://github.com/python/mypy) does it much better. Also, there are [plenty of alternatives](https://github.com/typeddjango/awesome-python-typing) for both static and dynamic validation. Deal is intended to empower types, to tell a bit more about possible values set than you can do with type annotations, not replace them. However, if you want to play with deal a bit or make types a part of contracts, [PySchemes](https://github.com/spy16/pyschemes)-based contract is the best choice: ```python import deal @@ -78,3 +78,7 @@ In short signature, `_` is a `dict` with access by attributes. Hence it has all ## What can be contract You can use any logic inside the validator. However, thumb up rule is to keep contracts [pure](https://en.wikipedia.org/wiki/Pure_function) (without any side-effects, even logging). The main motivation for it is that some contracts can be partially executed by [linter](../basic/linter.md). + +## Permissive license + +Deal distributed under [MIT License](https://en.wikipedia.org/wiki/MIT_License) which is a permissive license with high [license compatibility](https://en.wikipedia.org/wiki/License_compatibility). However, Deal has [astroid](https://github.com/PyCQA/astroid) in the dependencies which is licensed under [LGPL](https://en.wikipedia.org/wiki/GNU_Lesser_General_Public_License). While this license allows to be used in non-LGPL proprietary software too, it still can be not enough for some companies. So, if the legal department in your company forbids using LGPL libraries in transitive dependencies, you can freely remove `astroid` from the project dependencies before shipping it on the production. All CLI commands won't work anymore but runtime checks will. diff --git a/tests/test_imports.py b/tests/test_imports.py index 5df1e9b4..fb471b29 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -8,8 +8,21 @@ # project import deal -from deal._imports import DealLoader, deactivate -from deal.linter._extractors.common import get_name +from deal._imports import DealLoader, deactivate, get_name + + +@pytest.mark.parametrize('text, expected', [ + ('name', 'name'), + ('left.right', 'left.right'), + + ('left().right', None), + ('1', None), +]) +def test_get_name(text, expected): + tree = ast.parse(text) + print(ast.dump(tree)) + expr = tree.body[0].value + assert get_name(expr=expr) == expected def test_get_contracts(): diff --git a/tests/test_main.py b/tests/test_main.py index db41df8f..ea5ceb45 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -6,3 +6,21 @@ def test_cli_calling(): result = subprocess.run([sys.executable, '-m', 'deal', 'lint', __file__]) assert result.returncode == 0 + + +def test_do_not_import_linter(): + """When deal is imported, it must not trigger importing linter. + """ + for name in list(sys.modules): + if name == 'deal' or name.startswith('deal.'): + del sys.modules[name] + if name == 'astroid' or name.startswith('astroid.'): + del sys.modules[name] + + import deal # noqa: F401 + + for name in list(sys.modules): + if name.startswith('deal.lint'): + raise ImportError('oh no, linter was imported') + if name == 'astroid' or name.startswith('astroid.'): + raise ImportError('oh no, astroid was imported') From 7d6cd18877beb0a4c4127dc77bd70ce1174965a9 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 29 Sep 2020 14:01:18 +0200 Subject: [PATCH 29/30] test docs --- tests/test_docs.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/test_docs.py diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 00000000..b6ac360f --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,57 @@ +from pathlib import Path +from deal.linter._rules import rules, CheckMarkers +from deal._cli._main import COMMANDS + + +root = Path(__file__).parent.parent / 'docs' + + +def test_all_codes_listed(): + path = root / 'basic' / 'linter.md' + content = path.read_text() + for checker in rules: + if type(checker) is CheckMarkers: + continue + code = '| DEAL{:03d} |'.format(checker.code) + assert code in content + + for marker, code in CheckMarkers.codes.items(): + code = '| DEAL{:03d} | missed marker ({})'.format(code, marker) + assert code in content + + +def test_cli_included(): + path = root / 'details' / 'cli.md' + content = path.read_text() + for name, cmd in COMMANDS.items(): + # has header + tmpl = '## {n}\n\n' + line = tmpl.format(n=name) + assert line in content + + # has autodoc + tmpl = '```eval_rst\n.. autofunction:: deal._cli._{n}.{c}\n```' + line = tmpl.format(n=name, c=cmd.__name__) + assert line in content + + # header and autodoc go next to each other + tmpl = '## {n}\n\n```eval_rst\n.. autofunction:: deal._cli._{n}.{c}\n```' + line = tmpl.format(n=name, c=cmd.__name__) + assert line in content + + +def test_examples_included(): + path = root / 'details' / 'examples.md' + content = path.read_text() + for path in root.parent.joinpath('examples').iterdir(): + if '__' in path.name: + continue + # has header + tmpl = '## {}\n\n' + line = tmpl.format(path.stem) + assert line in content + + # has include + tmpl = '```eval_rst\n.. literalinclude:: ../../examples/{}\n```' + line = tmpl.format(path.name) + assert line in content From a60c3db13bb184b6ce08cdda59f83af76e0e8f08 Mon Sep 17 00:00:00 2001 From: Gram Date: Tue, 29 Sep 2020 14:36:17 +0200 Subject: [PATCH 30/30] restore sys.modules --- tests/test_main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index ea5ceb45..380df5c9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -11,6 +11,7 @@ def test_cli_calling(): def test_do_not_import_linter(): """When deal is imported, it must not trigger importing linter. """ + old_modules = sys.modules.copy() for name in list(sys.modules): if name == 'deal' or name.startswith('deal.'): del sys.modules[name] @@ -24,3 +25,5 @@ def test_do_not_import_linter(): raise ImportError('oh no, linter was imported') if name == 'astroid' or name.startswith('astroid.'): raise ImportError('oh no, astroid was imported') + + sys.modules = old_modules