From 5de325ed23df308266748728a8b5c12ae5ef44ec Mon Sep 17 00:00:00 2001 From: bcaller Date: Mon, 23 Jul 2018 17:13:47 +0100 Subject: [PATCH 1/4] Add __init__.py to modules as a package import from test_project.folder import ... should import module in test_project/folder/__init__.py and all the modules within the folder for working out if taint propagates. --- examples/test_project/folder/__init__.py | 0 pyt/core/project_handler.py | 19 +++++++++++-------- tests/core/project_handler_test.py | 8 ++++++-- 3 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 examples/test_project/folder/__init__.py diff --git a/examples/test_project/folder/__init__.py b/examples/test_project/folder/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyt/core/project_handler.py b/pyt/core/project_handler.py index 48eccfd1..0136010d 100644 --- a/pyt/core/project_handler.py +++ b/pyt/core/project_handler.py @@ -52,14 +52,10 @@ def get_modules(path): '.' ) directory = directory.replace('.', '', 1) - if directory: - modules.append( - ('.'.join((module_root, directory, filename.replace('.py', ''))), os.path.join(root, filename)) - ) - else: - modules.append( - ('.'.join((module_root, filename.replace('.py', ''))), os.path.join(root, filename)) - ) + modules.append(( + '.'.join(p for p in (module_root, directory, _filename_to_module(filename)) if p), + os.path.join(root, filename) + )) return modules @@ -68,3 +64,10 @@ def _is_python_file(path): if os.path.splitext(path)[1] == '.py': return True return False + + +def _filename_to_module(filename): + if filename == '__init__.py': + return '' + else: + return os.path.splitext(filename)[0] diff --git a/tests/core/project_handler_test.py b/tests/core/project_handler_test.py index 4de34b21..ddc812c7 100644 --- a/tests/core/project_handler_test.py +++ b/tests/core/project_handler_test.py @@ -32,6 +32,7 @@ def test_get_modules(self): utils_path = os.path.join(project_folder, 'utils.py') exceptions_path = os.path.join(project_folder, 'exceptions.py') some_path = os.path.join(project_folder, folder, 'some.py') + __init__path = os.path.join(project_folder, folder, '__init__.py') indhold_path = os.path.join(project_folder, folder, directory, 'indhold.py') # relative_folder_name = '.' + folder @@ -39,21 +40,24 @@ def test_get_modules(self): utils_name = project_namespace + '.' + 'utils' exceptions_name = project_namespace + '.' + 'exceptions' some_name = project_namespace + '.' + folder + '.some' + __init__name = project_namespace + '.' + folder indhold_name = project_namespace + '.' + folder + '.' + directory + '.indhold' app_tuple = (app_name, app_path) utils_tuple = (utils_name, utils_path) exceptions_tuple = (exceptions_name, exceptions_path) some_tuple = (some_name, some_path) + __init__tuple = (__init__name, __init__path) indhold_tuple = (indhold_name, indhold_path) self.assertIn(app_tuple, modules) self.assertIn(utils_tuple, modules) self.assertIn(exceptions_tuple, modules) self.assertIn(some_tuple, modules) + self.assertIn(__init__tuple, modules) self.assertIn(indhold_tuple, modules) - self.assertEqual(len(modules), 5) + self.assertEqual(len(modules), 6) def test_get_modules_and_packages(self): project_folder = os.path.normpath(os.path.join('examples', 'test_project')) @@ -104,4 +108,4 @@ def test_get_modules_and_packages(self): self.assertIn(some_tuple, modules) self.assertIn(indhold_tuple, modules) - self.assertEqual(len(modules), 7) + self.assertEqual(len(modules), 8) From e2ad3b03da8132c49e4d3c75d0301a10c839c5dc Mon Sep 17 00:00:00 2001 From: bcaller Date: Mon, 23 Jul 2018 17:09:27 +0100 Subject: [PATCH 2/4] Imports: --dont-prepend-root flag For a project with root in /app, currently pyt expects that all of the imports are of the form: from app.folder.module import thing But actually a project could not be expecting the module root to be prepended. My projects use: from folder.module import thing This adds an optional boolean flag to change the behaviour of get_modules. --- pyt/__main__.py | 2 +- pyt/core/project_handler.py | 27 +++++++++++++++------------ pyt/usage.py | 7 +++++++ tests/core/project_handler_test.py | 28 +++++++++++++++++++++++++++- tests/usage_test.py | 8 ++++++-- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/pyt/__main__.py b/pyt/__main__.py index a979eb6e..1d76828e 100644 --- a/pyt/__main__.py +++ b/pyt/__main__.py @@ -86,7 +86,7 @@ def main(command_line_args=sys.argv[1:]): # noqa: C901 directory = os.path.normpath(args.project_root) else: directory = os.path.dirname(path) - project_modules = get_modules(directory) + project_modules = get_modules(directory, prepend_module_root=args.prepend_module_root) local_modules = get_directory_modules(directory) tree = generate_ast(path) diff --git a/pyt/core/project_handler.py b/pyt/core/project_handler.py index 0136010d..f6a30f0e 100644 --- a/pyt/core/project_handler.py +++ b/pyt/core/project_handler.py @@ -31,7 +31,7 @@ def get_directory_modules(directory): return _local_modules -def get_modules(path): +def get_modules(path, prepend_module_root=True): """Return a list containing tuples of e.g. ('test_project.utils', 'example/test_project/utils.py') """ @@ -52,10 +52,20 @@ def get_modules(path): '.' ) directory = directory.replace('.', '', 1) - modules.append(( - '.'.join(p for p in (module_root, directory, _filename_to_module(filename)) if p), - os.path.join(root, filename) - )) + + module_name_parts = [] + if prepend_module_root: + module_name_parts.append(module_root) + if directory: + module_name_parts.append(directory) + + if filename == '__init__.py': + path = root + else: + module_name_parts.append(os.path.splitext(filename)[0]) + path = os.path.join(root, filename) + + modules.append(('.'.join(module_name_parts), path)) return modules @@ -64,10 +74,3 @@ def _is_python_file(path): if os.path.splitext(path)[1] == '.py': return True return False - - -def _filename_to_module(filename): - if filename == '__init__.py': - return '' - else: - return os.path.splitext(filename)[0] diff --git a/pyt/usage.py b/pyt/usage.py index 30286215..82d15027 100644 --- a/pyt/usage.py +++ b/pyt/usage.py @@ -101,6 +101,13 @@ def _add_optional_group(parser): default='', help='Separate files with commas' ) + optional_group.add_argument( + '--dont-prepend-root', + help="In project root e.g. /app, imports are not prepended with app.*", + action='store_false', + default=True, + dest='prepend_module_root' + ) def _add_print_group(parser): diff --git a/tests/core/project_handler_test.py b/tests/core/project_handler_test.py index ddc812c7..04e84be2 100644 --- a/tests/core/project_handler_test.py +++ b/tests/core/project_handler_test.py @@ -32,7 +32,7 @@ def test_get_modules(self): utils_path = os.path.join(project_folder, 'utils.py') exceptions_path = os.path.join(project_folder, 'exceptions.py') some_path = os.path.join(project_folder, folder, 'some.py') - __init__path = os.path.join(project_folder, folder, '__init__.py') + __init__path = os.path.join(project_folder, folder) indhold_path = os.path.join(project_folder, folder, directory, 'indhold.py') # relative_folder_name = '.' + folder @@ -59,6 +59,32 @@ def test_get_modules(self): self.assertEqual(len(modules), 6) + def test_get_modules_no_prepend_root(self): + project_folder = os.path.normpath(os.path.join('examples', 'test_project')) + + folder = 'folder' + directory = 'directory' + + modules = get_modules(project_folder, prepend_module_root=False) + + app_path = os.path.join(project_folder, 'app.py') + __init__path = os.path.join(project_folder, folder) + indhold_path = os.path.join(project_folder, folder, directory, 'indhold.py') + + app_name = 'app' + __init__name = folder + indhold_name = folder + '.' + directory + '.indhold' + + app_tuple = (app_name, app_path) + __init__tuple = (__init__name, __init__path) + indhold_tuple = (indhold_name, indhold_path) + + self.assertIn(app_tuple, modules) + self.assertIn(__init__tuple, modules) + self.assertIn(indhold_tuple, modules) + + self.assertEqual(len(modules), 6) + def test_get_modules_and_packages(self): project_folder = os.path.normpath(os.path.join('examples', 'test_project')) diff --git a/tests/usage_test.py b/tests/usage_test.py index d9ed7cec..a5e7db4b 100644 --- a/tests/usage_test.py +++ b/tests/usage_test.py @@ -28,7 +28,8 @@ def test_no_args(self): EXPECTED = """usage: python -m pyt [-h] [-a ADAPTOR] [-pr PROJECT_ROOT] [-b BASELINE_JSON_FILE] [-j] [-m BLACKBOX_MAPPING_FILE] [-t TRIGGER_WORD_FILE] [-o OUTPUT_FILE] [--ignore-nosec] - [-r] [-x EXCLUDED_PATHS] [-trim] [-i] + [-r] [-x EXCLUDED_PATHS] [--dont-prepend-root] [-trim] + [-i] targets [targets ...] required arguments: @@ -55,6 +56,8 @@ def test_no_args(self): -r, --recursive find and process files in subdirectories -x EXCLUDED_PATHS, --exclude EXCLUDED_PATHS Separate files with commas + --dont-prepend-root In project root e.g. /app, imports are not prepended + with app.* print arguments: -trim, --trim-reassigned-in @@ -73,7 +76,8 @@ def test_valid_args_but_no_targets(self): EXPECTED = """usage: python -m pyt [-h] [-a ADAPTOR] [-pr PROJECT_ROOT] [-b BASELINE_JSON_FILE] [-j] [-m BLACKBOX_MAPPING_FILE] [-t TRIGGER_WORD_FILE] [-o OUTPUT_FILE] [--ignore-nosec] - [-r] [-x EXCLUDED_PATHS] [-trim] [-i] + [-r] [-x EXCLUDED_PATHS] [--dont-prepend-root] [-trim] + [-i] targets [targets ...] python -m pyt: error: the following arguments are required: targets\n""" From 11bcd2d93f45c28c89042f3ae82e212585d598f5 Mon Sep 17 00:00:00 2001 From: bcaller Date: Mon, 23 Jul 2018 17:44:30 +0100 Subject: [PATCH 3/4] Remove unused function: valid_date --- .coveragerc | 1 - pyt/usage.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/.coveragerc b/.coveragerc index c7e7a385..df3137cf 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,6 @@ show_missing = True exclude_lines = - def valid_date def __repr__ def __str__ if __name__ == .__main__.: diff --git a/pyt/usage.py b/pyt/usage.py index 82d15027..6a49bb3c 100644 --- a/pyt/usage.py +++ b/pyt/usage.py @@ -18,15 +18,6 @@ ) -def valid_date(s): - date_format = "%Y-%m-%d" - try: - return datetime.strptime(s, date_format).date() - except ValueError: - msg = "Not a valid date: '{0}'. Format: {1}".format(s, date_format) - raise argparse.ArgumentTypeError(msg) - - def _add_required_group(parser): required_group = parser.add_argument_group('required arguments') required_group.add_argument( From 11bb85b90fc1376048145055c78b1f3eb93a8c1b Mon Sep 17 00:00:00 2001 From: bcaller Date: Tue, 24 Jul 2018 14:01:47 +0100 Subject: [PATCH 4/4] Imports: --no-local-imports flag I had a problem where one of my folders had a file called flask.py. Because of this, any imports of the package flask were causing pyt to import from the local file flask.py. In my project this caused a circular import and RecursionError which crashed pyt. Adds a flag so that imports relative to the project root still work: from some.directory.flask import ... but from flask import ... will now only import from project root or treat as an IgnoredNode(). Relative imports: from .flask import ... are not affected and will still work. --- pyt/__main__.py | 3 ++- pyt/cfg/expr_visitor.py | 6 ++++-- pyt/cfg/make_cfg.py | 6 ++++-- pyt/cfg/stmt_visitor.py | 5 ++++- pyt/usage.py | 8 ++++++++ tests/usage_test.py | 11 +++++++---- 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pyt/__main__.py b/pyt/__main__.py index 1d76828e..5eed4747 100644 --- a/pyt/__main__.py +++ b/pyt/__main__.py @@ -94,7 +94,8 @@ def main(command_line_args=sys.argv[1:]): # noqa: C901 tree, project_modules, local_modules, - path + path, + allow_local_directory_imports=args.allow_local_imports ) cfg_list = [cfg] diff --git a/pyt/cfg/expr_visitor.py b/pyt/cfg/expr_visitor.py index b4a96168..f4d99d6a 100644 --- a/pyt/cfg/expr_visitor.py +++ b/pyt/cfg/expr_visitor.py @@ -37,11 +37,13 @@ def __init__( project_modules, local_modules, filename, - module_definitions=None + module_definitions=None, + allow_local_directory_imports=True ): """Create an empty CFG.""" + super().__init__(allow_local_directory_imports=allow_local_directory_imports) self.project_modules = project_modules - self.local_modules = local_modules + self.local_modules = local_modules if self._allow_local_modules else [] self.filenames = [filename] self.blackbox_assignments = set() self.nodes = list() diff --git a/pyt/cfg/make_cfg.py b/pyt/cfg/make_cfg.py index a60b734e..65aee3a8 100644 --- a/pyt/cfg/make_cfg.py +++ b/pyt/cfg/make_cfg.py @@ -30,14 +30,16 @@ def make_cfg( project_modules, local_modules, filename, - module_definitions=None + module_definitions=None, + allow_local_directory_imports=True ): visitor = ExprVisitor( tree, project_modules, local_modules, filename, - module_definitions + module_definitions, + allow_local_directory_imports ) return CFG( visitor.nodes, diff --git a/pyt/cfg/stmt_visitor.py b/pyt/cfg/stmt_visitor.py index ba463c32..7bbafb09 100644 --- a/pyt/cfg/stmt_visitor.py +++ b/pyt/cfg/stmt_visitor.py @@ -53,6 +53,9 @@ class StmtVisitor(ast.NodeVisitor): + def __init__(self, allow_local_directory_imports=True): + self._allow_local_modules = allow_local_directory_imports + super().__init__() def visit_Module(self, node): return self.stmt_star_handler(node.body) @@ -753,7 +756,7 @@ def add_module( # noqa: C901 # Analyse the file self.filenames.append(module_path) - self.local_modules = get_directory_modules(module_path) + self.local_modules = get_directory_modules(module_path) if self._allow_local_modules else [] tree = generate_ast(module_path) # module[0] is None during e.g. "from . import foo", so we must str() diff --git a/pyt/usage.py b/pyt/usage.py index 6a49bb3c..a4ec5d81 100644 --- a/pyt/usage.py +++ b/pyt/usage.py @@ -99,6 +99,14 @@ def _add_optional_group(parser): default=True, dest='prepend_module_root' ) + optional_group.add_argument( + '--no-local-imports', + help='If set, absolute imports must be relative to the project root. ' + 'If not set, modules in the same directory can be imported just by their names.', + action='store_false', + default=True, + dest='allow_local_imports' + ) def _add_print_group(parser): diff --git a/tests/usage_test.py b/tests/usage_test.py index a5e7db4b..027c6f00 100644 --- a/tests/usage_test.py +++ b/tests/usage_test.py @@ -28,8 +28,8 @@ def test_no_args(self): EXPECTED = """usage: python -m pyt [-h] [-a ADAPTOR] [-pr PROJECT_ROOT] [-b BASELINE_JSON_FILE] [-j] [-m BLACKBOX_MAPPING_FILE] [-t TRIGGER_WORD_FILE] [-o OUTPUT_FILE] [--ignore-nosec] - [-r] [-x EXCLUDED_PATHS] [--dont-prepend-root] [-trim] - [-i] + [-r] [-x EXCLUDED_PATHS] [--dont-prepend-root] + [--no-local-imports] [-trim] [-i] targets [targets ...] required arguments: @@ -58,6 +58,9 @@ def test_no_args(self): Separate files with commas --dont-prepend-root In project root e.g. /app, imports are not prepended with app.* + --no-local-imports If set, absolute imports must be relative to the + project root. If not set, modules in the same + directory can be imported just by their names. print arguments: -trim, --trim-reassigned-in @@ -76,8 +79,8 @@ def test_valid_args_but_no_targets(self): EXPECTED = """usage: python -m pyt [-h] [-a ADAPTOR] [-pr PROJECT_ROOT] [-b BASELINE_JSON_FILE] [-j] [-m BLACKBOX_MAPPING_FILE] [-t TRIGGER_WORD_FILE] [-o OUTPUT_FILE] [--ignore-nosec] - [-r] [-x EXCLUDED_PATHS] [--dont-prepend-root] [-trim] - [-i] + [-r] [-x EXCLUDED_PATHS] [--dont-prepend-root] + [--no-local-imports] [-trim] [-i] targets [targets ...] python -m pyt: error: the following arguments are required: targets\n"""