From 0d6fa20012035ee29858b05d80779cae6ffc99ff Mon Sep 17 00:00:00 2001 From: Lenny Truong Date: Thu, 22 Aug 2019 11:44:08 -0700 Subject: [PATCH 1/7] Add inspect debug info --- ast_tools/passes/base.py | 1 + ast_tools/passes/debug.py | 18 ++++++++++++++++++ ast_tools/passes/util.py | 13 ++++++++++--- tests/test_passes.py | 25 ++++++++++++++++++++++++- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/ast_tools/passes/base.py b/ast_tools/passes/base.py index 4462f13..f76dfaa 100644 --- a/ast_tools/passes/base.py +++ b/ast_tools/passes/base.py @@ -21,6 +21,7 @@ def __call__(self, args: PASS_ARGS_T) -> PASS_ARGS_T: def rewrite(self, env: SymbolTable, tree: ast.AST, + metadata: dict, ) -> PASS_ARGS_T: """ diff --git a/ast_tools/passes/debug.py b/ast_tools/passes/debug.py index 8d677e8..abc63d5 100644 --- a/ast_tools/passes/debug.py +++ b/ast_tools/passes/debug.py @@ -17,6 +17,8 @@ def __init__(self, dump_env: bool = False, file: tp.Optional[str] = None, append: tp.Optional[bool] = None, + dump_source_filename: bool = False, + dump_source_lines: bool = False ) -> PASS_ARGS_T: self.dump_ast = dump_ast self.dump_src = dump_src @@ -25,10 +27,13 @@ def __init__(self, warnings.warn('Option append has no effect when file is None', stacklevel=2) self.file = file self.append = append + self.dump_source_filename = dump_source_filename + self.dump_source_lines = dump_source_lines def rewrite(self, tree: ast.AST, env: SymbolTable, + metadata: dict, ) -> PASS_ARGS_T: def _do_dumps(dumps, dump_writer): @@ -44,6 +49,19 @@ def _do_dumps(dumps, dump_writer): dumps.append(('SRC', astor.to_source(tree))) if self.dump_env: dumps.append(('ENV', repr(env))) + if self.dump_source_filename: + if "source_filename" not in metadata: + raise Exception("Cannot dump source filename without " + "@begin_rewrite(debug=True)") + dumps.append(('SOURCE_FILENAME', metadata["source_filename"])) + if self.dump_source_lines: + if "source_lines" not in metadata: + raise Exception("Cannot dump source lines without " + "@begin_rewrite(debug=True)") + lines, start_line_number = metadata["source_lines"] + dump_str = "".join(f"{start_line_number + i}:{line}" for i, line in + enumerate(lines)) + dumps.append(('SOURCE_LINES', dump_str)) if self.file is not None: if self.append: diff --git a/ast_tools/passes/util.py b/ast_tools/passes/util.py index e6c1775..f589b2b 100644 --- a/ast_tools/passes/util.py +++ b/ast_tools/passes/util.py @@ -1,3 +1,4 @@ +import inspect import ast import typing as tp @@ -13,13 +14,18 @@ class begin_rewrite: """ begins a chain of passes """ - def __init__(self): + def __init__(self, debug=False): env = get_symbol_table([self.__init__]) self.env = env + self.debug = debug def __call__(self, fn) -> PASS_ARGS_T: tree = get_ast(fn) - return tree, self.env + metadata = {} + if self.debug: + metadata["source_filename"] = inspect.getsourcefile(fn) + metadata["source_lines"] = inspect.getsourcelines(fn) + return tree, self.env, metadata def _issubclass(t, s) -> bool: try: @@ -37,7 +43,8 @@ def __init__(self, path: tp.Optional[str] = None): def rewrite(self, tree: ast.AST, - env: SymbolTable) -> tp.Union[tp.Callable, type]: + env: SymbolTable, + metadata: dict) -> tp.Union[tp.Callable, type]: decorators = [] first_group = True in_group = False diff --git a/tests/test_passes.py b/tests/test_passes.py index c7b5b83..71a360f 100644 --- a/tests/test_passes.py +++ b/tests/test_passes.py @@ -1,7 +1,7 @@ import functools import inspect -from ast_tools.passes import begin_rewrite, end_rewrite +from ast_tools.passes import begin_rewrite, end_rewrite, debug @@ -25,3 +25,26 @@ def foo(): def foo(): pass ''' + + +def test_debug(capsys): + + @end_rewrite + @debug(dump_source_filename=True, dump_source_lines=True) + @begin_rewrite(debug=True) + def foo(): + print("bar") + assert capsys.readouterr().out == """\ +BEGIN SOURCE_FILENAME +/Users/leonardtruong/repos/ast_tools/tests/test_passes.py +END SOURCE_FILENAME + +BEGIN SOURCE_LINES +32: @end_rewrite +33: @debug(dump_source_filename=True, dump_source_lines=True) +34: @begin_rewrite(debug=True) +35: def foo(): +36: print("bar") +END SOURCE_LINES + +""" From 72afaa5389013b2fc40c5375c35445b92c8ce665 Mon Sep 17 00:00:00 2001 From: Lenny Truong Date: Thu, 22 Aug 2019 11:48:16 -0700 Subject: [PATCH 2/7] Update PASS_ARGS_T logic --- ast_tools/passes/base.py | 2 +- ast_tools/passes/debug.py | 2 +- ast_tools/passes/ssa.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ast_tools/passes/base.py b/ast_tools/passes/base.py index f76dfaa..353b523 100644 --- a/ast_tools/passes/base.py +++ b/ast_tools/passes/base.py @@ -6,7 +6,7 @@ __ALL__ = ['Pass', 'PASS_ARGS_T'] -PASS_ARGS_T = tp.Tuple[ast.AST, SymbolTable] +PASS_ARGS_T = tp.Tuple[ast.AST, SymbolTable, dict] class Pass(metaclass=ABCMeta): """ diff --git a/ast_tools/passes/debug.py b/ast_tools/passes/debug.py index abc63d5..9396c59 100644 --- a/ast_tools/passes/debug.py +++ b/ast_tools/passes/debug.py @@ -74,4 +74,4 @@ def _do_dumps(dumps, dump_writer): def _print(*args, **kwargs): print(*args, end='', **kwargs) _do_dumps(dumps, _print) - return tree, env + return tree, env, metadata diff --git a/ast_tools/passes/ssa.py b/ast_tools/passes/ssa.py index b7ec360..f560f31 100644 --- a/ast_tools/passes/ssa.py +++ b/ast_tools/passes/ssa.py @@ -342,7 +342,7 @@ class ssa(Pass): def __init__(self, return_prefix: str = '__return_value'): self.return_prefix = return_prefix - def rewrite(self, tree: ast.AST, env: SymbolTable): + def rewrite(self, tree: ast.AST, env: SymbolTable, metadata: dict): if not isinstance(tree, ast.FunctionDef): raise TypeError('ssa should only be applied to functions') r_name = gen_free_prefix(tree, env, self.return_prefix) @@ -353,4 +353,4 @@ def rewrite(self, tree: ast.AST, env: SymbolTable): value=_build_return(visitor.returns) ) ) - return tree, env + return tree, env, metadata From 56d33e2545c0f5e06e0fc018ab4bced6c3fa390a Mon Sep 17 00:00:00 2001 From: Lenny Truong Date: Thu, 22 Aug 2019 11:50:49 -0700 Subject: [PATCH 3/7] Use absolute path of test file --- tests/test_passes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_passes.py b/tests/test_passes.py index 71a360f..8e03e57 100644 --- a/tests/test_passes.py +++ b/tests/test_passes.py @@ -34,9 +34,9 @@ def test_debug(capsys): @begin_rewrite(debug=True) def foo(): print("bar") - assert capsys.readouterr().out == """\ + assert capsys.readouterr().out == f"""\ BEGIN SOURCE_FILENAME -/Users/leonardtruong/repos/ast_tools/tests/test_passes.py +{os.path.abspath(__file__)} END SOURCE_FILENAME BEGIN SOURCE_LINES From 879eabe6d7138773a7925417f2694821b0e965de Mon Sep 17 00:00:00 2001 From: Lenny Truong Date: Thu, 22 Aug 2019 11:51:13 -0700 Subject: [PATCH 4/7] Import os --- tests/test_passes.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_passes.py b/tests/test_passes.py index 8e03e57..9dd67fe 100644 --- a/tests/test_passes.py +++ b/tests/test_passes.py @@ -1,3 +1,4 @@ +import os import functools import inspect @@ -40,11 +41,11 @@ def foo(): END SOURCE_FILENAME BEGIN SOURCE_LINES -32: @end_rewrite -33: @debug(dump_source_filename=True, dump_source_lines=True) -34: @begin_rewrite(debug=True) -35: def foo(): -36: print("bar") +33: @end_rewrite +34: @debug(dump_source_filename=True, dump_source_lines=True) +35: @begin_rewrite(debug=True) +36: def foo(): +37: print("bar") END SOURCE_LINES """ From 0e115596370be140d3fcd5689edd1352238b05b1 Mon Sep 17 00:00:00 2001 From: Lenny Truong Date: Thu, 22 Aug 2019 11:56:56 -0700 Subject: [PATCH 5/7] Test error conditions --- tests/test_passes.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_passes.py b/tests/test_passes.py index 9dd67fe..14d5b9f 100644 --- a/tests/test_passes.py +++ b/tests/test_passes.py @@ -49,3 +49,24 @@ def foo(): END SOURCE_LINES """ + + +def test_debug_error(): + + try: + @end_rewrite + @debug(dump_source_filename=True) + @begin_rewrite() + def foo(): + print("bar") + except Exception as e: + assert str(e) == "Cannot dump source filename without @begin_rewrite(debug=True)" + + try: + @end_rewrite + @debug(dump_source_lines=True) + @begin_rewrite() + def foo(): + print("bar") + except Exception as e: + assert str(e) == "Cannot dump source lines without @begin_rewrite(debug=True)" From cc168a29d6b7e3c7ce6a3cc0653aa1487ca95447 Mon Sep 17 00:00:00 2001 From: Lenny Truong Date: Thu, 22 Aug 2019 14:50:54 -0700 Subject: [PATCH 6/7] dict -> MutableMapping --- ast_tools/passes/base.py | 5 +++-- ast_tools/passes/debug.py | 3 ++- ast_tools/passes/ssa.py | 4 ++-- ast_tools/passes/util.py | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ast_tools/passes/base.py b/ast_tools/passes/base.py index 353b523..aad40d5 100644 --- a/ast_tools/passes/base.py +++ b/ast_tools/passes/base.py @@ -1,4 +1,5 @@ from abc import ABCMeta, abstractmethod +from collections import MutableMapping import ast import typing as tp @@ -6,7 +7,7 @@ __ALL__ = ['Pass', 'PASS_ARGS_T'] -PASS_ARGS_T = tp.Tuple[ast.AST, SymbolTable, dict] +PASS_ARGS_T = tp.Tuple[ast.AST, SymbolTable, MutableMapping] class Pass(metaclass=ABCMeta): """ @@ -21,7 +22,7 @@ def __call__(self, args: PASS_ARGS_T) -> PASS_ARGS_T: def rewrite(self, env: SymbolTable, tree: ast.AST, - metadata: dict, + metadata: MutableMapping, ) -> PASS_ARGS_T: """ diff --git a/ast_tools/passes/debug.py b/ast_tools/passes/debug.py index 9396c59..5ec7b23 100644 --- a/ast_tools/passes/debug.py +++ b/ast_tools/passes/debug.py @@ -1,6 +1,7 @@ import ast import typing as tp import warnings +from collections import MutableMapping import astor @@ -33,7 +34,7 @@ def __init__(self, def rewrite(self, tree: ast.AST, env: SymbolTable, - metadata: dict, + metadata: MutableMapping, ) -> PASS_ARGS_T: def _do_dumps(dumps, dump_writer): diff --git a/ast_tools/passes/ssa.py b/ast_tools/passes/ssa.py index f560f31..1ff7e1f 100644 --- a/ast_tools/passes/ssa.py +++ b/ast_tools/passes/ssa.py @@ -1,5 +1,5 @@ import ast -from collections import ChainMap, Counter +from collections import ChainMap, Counter, MutableMapping import itertools import warnings import weakref @@ -342,7 +342,7 @@ class ssa(Pass): def __init__(self, return_prefix: str = '__return_value'): self.return_prefix = return_prefix - def rewrite(self, tree: ast.AST, env: SymbolTable, metadata: dict): + def rewrite(self, tree: ast.AST, env: SymbolTable, metadata: MutableMapping): if not isinstance(tree, ast.FunctionDef): raise TypeError('ssa should only be applied to functions') r_name = gen_free_prefix(tree, env, self.return_prefix) diff --git a/ast_tools/passes/util.py b/ast_tools/passes/util.py index f589b2b..07a0a60 100644 --- a/ast_tools/passes/util.py +++ b/ast_tools/passes/util.py @@ -1,6 +1,7 @@ import inspect import ast import typing as tp +from collections import MutableMapping from . import Pass from . import PASS_ARGS_T @@ -44,7 +45,7 @@ def __init__(self, path: tp.Optional[str] = None): def rewrite(self, tree: ast.AST, env: SymbolTable, - metadata: dict) -> tp.Union[tp.Callable, type]: + metadata: MutableMapping) -> tp.Union[tp.Callable, type]: decorators = [] first_group = True in_group = False From 60e18b7ae795a358c75d7ba3eb64db67a39fe0fa Mon Sep 17 00:00:00 2001 From: Caleb Donovick Date: Thu, 22 Aug 2019 14:57:24 -0700 Subject: [PATCH 7/7] Remove deprecated import --- ast_tools/passes/base.py | 5 ++--- ast_tools/passes/debug.py | 3 +-- ast_tools/passes/ssa.py | 4 ++-- ast_tools/passes/util.py | 3 +-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ast_tools/passes/base.py b/ast_tools/passes/base.py index aad40d5..ff18c10 100644 --- a/ast_tools/passes/base.py +++ b/ast_tools/passes/base.py @@ -1,5 +1,4 @@ from abc import ABCMeta, abstractmethod -from collections import MutableMapping import ast import typing as tp @@ -7,7 +6,7 @@ __ALL__ = ['Pass', 'PASS_ARGS_T'] -PASS_ARGS_T = tp.Tuple[ast.AST, SymbolTable, MutableMapping] +PASS_ARGS_T = tp.Tuple[ast.AST, SymbolTable, tp.MutableMapping] class Pass(metaclass=ABCMeta): """ @@ -22,7 +21,7 @@ def __call__(self, args: PASS_ARGS_T) -> PASS_ARGS_T: def rewrite(self, env: SymbolTable, tree: ast.AST, - metadata: MutableMapping, + metadata: tp.MutableMapping, ) -> PASS_ARGS_T: """ diff --git a/ast_tools/passes/debug.py b/ast_tools/passes/debug.py index 5ec7b23..bb34996 100644 --- a/ast_tools/passes/debug.py +++ b/ast_tools/passes/debug.py @@ -1,7 +1,6 @@ import ast import typing as tp import warnings -from collections import MutableMapping import astor @@ -34,7 +33,7 @@ def __init__(self, def rewrite(self, tree: ast.AST, env: SymbolTable, - metadata: MutableMapping, + metadata: tp.MutableMapping, ) -> PASS_ARGS_T: def _do_dumps(dumps, dump_writer): diff --git a/ast_tools/passes/ssa.py b/ast_tools/passes/ssa.py index 1ff7e1f..6841547 100644 --- a/ast_tools/passes/ssa.py +++ b/ast_tools/passes/ssa.py @@ -1,5 +1,5 @@ import ast -from collections import ChainMap, Counter, MutableMapping +from collections import ChainMap, Counter import itertools import warnings import weakref @@ -342,7 +342,7 @@ class ssa(Pass): def __init__(self, return_prefix: str = '__return_value'): self.return_prefix = return_prefix - def rewrite(self, tree: ast.AST, env: SymbolTable, metadata: MutableMapping): + def rewrite(self, tree: ast.AST, env: SymbolTable, metadata: tp.MutableMapping): if not isinstance(tree, ast.FunctionDef): raise TypeError('ssa should only be applied to functions') r_name = gen_free_prefix(tree, env, self.return_prefix) diff --git a/ast_tools/passes/util.py b/ast_tools/passes/util.py index 07a0a60..855439d 100644 --- a/ast_tools/passes/util.py +++ b/ast_tools/passes/util.py @@ -1,7 +1,6 @@ import inspect import ast import typing as tp -from collections import MutableMapping from . import Pass from . import PASS_ARGS_T @@ -45,7 +44,7 @@ def __init__(self, path: tp.Optional[str] = None): def rewrite(self, tree: ast.AST, env: SymbolTable, - metadata: MutableMapping) -> tp.Union[tp.Callable, type]: + metadata: tp.MutableMapping) -> tp.Union[tp.Callable, type]: decorators = [] first_group = True in_group = False