From 5d0e335508909ef335087e932b9771984ecb1669 Mon Sep 17 00:00:00 2001 From: Giorgos Drosos Date: Mon, 15 May 2023 18:21:16 +0300 Subject: [PATCH 1/7] Add github action for linting --- .github/workflows/linters.yml | 59 +++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/linters.yml diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 0000000..d1aa5cf --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,59 @@ +name: Linters + +on: + pull_request: + push: + branches: + - main + + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + flake8: + name: flake8 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: python -m pip install flake8 + - name: flake8 + # Pinned to v2.0.0. + uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2 + with: + linters: flake8 + run: flake8 + + isort: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: python -m pip install isort + - name: isort + # Pinned to v2.0.0. + uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2 + with: + linters: isort + run: isort --check --diff pycg micro-benchmark-key-errs micro-benchmark + + black: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: black + uses: psf/black@stable \ No newline at end of file From 6edd91f2daeb55dfa29b97b929d0f934e3dd8b6b Mon Sep 17 00:00:00 2001 From: Giorgos Drosos Date: Tue, 16 May 2023 13:35:14 +0300 Subject: [PATCH 2/7] Disable linting in benchmark directories --- .github/workflows/linters.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index d1aa5cf..29a427c 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -31,7 +31,7 @@ jobs: uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2 with: linters: flake8 - run: flake8 + run: flake8 pycg isort: runs-on: ubuntu-latest @@ -48,7 +48,7 @@ jobs: uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2 with: linters: isort - run: isort --check --diff pycg micro-benchmark-key-errs micro-benchmark + run: isort --check --diff pycg black: runs-on: ubuntu-latest From 16204305ba442c37b206f684d2a44c66acb6fd97 Mon Sep 17 00:00:00 2001 From: Giorgos Drosos Date: Tue, 16 May 2023 14:45:12 +0300 Subject: [PATCH 3/7] Fix linting errors --- pycg/__main__.py | 28 ++++++++++----- pycg/formats/__init__.py | 2 +- pycg/formats/base.py | 4 ++- pycg/formats/fasten.py | 41 +++++++++++++-------- pycg/machinery/classes.py | 2 +- pycg/machinery/definitions.py | 31 ++++++++++------ pycg/machinery/imports.py | 6 ++-- pycg/machinery/key_err.py | 3 +- pycg/machinery/scopes.py | 3 +- pycg/processing/base.py | 42 +++++++++++++++------- pycg/processing/cgprocessor.py | 38 ++++++++++++-------- pycg/processing/keyerrprocessor.py | 3 +- pycg/processing/postprocessor.py | 37 ++++++++++++------- pycg/processing/preprocessor.py | 30 +++++++++++----- pycg/tests/definitions_test.py | 6 ++-- pycg/tests/fasten_format_test.py | 58 ++++++++++++++++++++---------- pycg/tests/imports_test.py | 19 ++++++---- pycg/tests/pointers_test.py | 7 ++-- pycg/tests/scopes_test.py | 45 +++++++++++++++-------- 19 files changed, 272 insertions(+), 133 deletions(-) diff --git a/pycg/__main__.py b/pycg/__main__.py index a109308..3b5517c 100644 --- a/pycg/__main__.py +++ b/pycg/__main__.py @@ -8,9 +8,12 @@ def main(): parser = argparse.ArgumentParser() - parser.add_argument("entry_point", nargs="*", help="Entry points to be processed") + parser.add_argument("entry_point", + nargs="*", + help="Entry points to be processed") parser.add_argument( - "--package", help="Package containing the code to be analyzed", default=None + "--package", + help="Package containing the code to be analyzed", default=None ) parser.add_argument( "--fasten", @@ -18,13 +21,17 @@ def main(): action="store_true", default=False, ) - parser.add_argument("--product", help="Package name", default="") + parser.add_argument("--product", + help="Package name", default="") parser.add_argument( - "--forge", help="Source the product was downloaded from", default="" + "--forge", + help="Source the product was downloaded from", default="" ) - parser.add_argument("--version", help="Version of the product", default="") + parser.add_argument("--version", + help="Version of the product", default="") parser.add_argument( - "--timestamp", help="Timestamp of the package's version", default=0 + "--timestamp", + help="Timestamp of the package's version", default=0 ) parser.add_argument( "--max-iter", @@ -52,9 +59,11 @@ def main(): ) parser.add_argument( - "--as-graph-output", help="Output for the assignment graph", default=None + "--as-graph-output", + help="Output for the assignment graph", default=None ) - parser.add_argument("-o", "--output", help="Output path", default=None) + parser.add_argument("-o", "--output", + help="Output path", default=None) args = parser.parse_args() @@ -66,7 +75,8 @@ def main(): if args.operation == CALL_GRAPH_OP: if args.fasten: formatter = formats.Fasten( - cg, args.package, args.product, args.forge, args.version, args.timestamp + cg, args.package, args.product, + args.forge, args.version, args.timestamp ) else: formatter = formats.Simple(cg) diff --git a/pycg/formats/__init__.py b/pycg/formats/__init__.py index 3b69969..813810a 100644 --- a/pycg/formats/__init__.py +++ b/pycg/formats/__init__.py @@ -18,6 +18,6 @@ # specific language governing permissions and limitations # under the License. # +from .as_graph import AsGraph from .fasten import Fasten from .simple import Simple -from .as_graph import AsGraph diff --git a/pycg/formats/base.py b/pycg/formats/base.py index 88f13bd..cb4c0cc 100644 --- a/pycg/formats/base.py +++ b/pycg/formats/base.py @@ -20,4 +20,6 @@ # class BaseFormatter: def generate(self): - raise NotImplementedError("generate method not implemented in child class") + raise NotImplementedError( + "generate method not implemented in child class" + ) diff --git a/pycg/formats/fasten.py b/pycg/formats/fasten.py index 3e98494..adfbcbf 100644 --- a/pycg/formats/fasten.py +++ b/pycg/formats/fasten.py @@ -28,7 +28,8 @@ class Fasten(BaseFormatter): - def __init__(self, cg_generator, package, product, forge, version, timestamp): + def __init__(self, cg_generator, package, product, forge, version, + timestamp): self.cg_generator = cg_generator self.internal_mods = self.cg_generator.output_internal_mods() or {} self.external_mods = self.cg_generator.output_external_mods() or {} @@ -55,9 +56,10 @@ def to_uri(self, modname, name=""): cleared = "" else: if not name.startswith(modname + "."): - raise Exception("name should start with modname", name, modname) + raise Exception("name should start with modname", name, + modname) - cleared = name[len(modname) + 1 :] + cleared = name[len(modname) + 1:] suffix = "" if name in self.functions: @@ -67,7 +69,7 @@ def to_uri(self, modname, name=""): def to_external_uri(self, modname, name=""): if modname == utils.constants.BUILTIN_NAME: - name = name[len(modname) + 1 :] + name = name[len(modname) + 1:] modname = ".builtin" return "//{}//{}".format(modname.replace("-", "_"), name) @@ -91,7 +93,8 @@ def find_dependencies(self, package_path): try: req = Requirement.parse(line) except ValueError: - # The specific line in the requirements.txt does not follow the Requirements File Format + # The specific line in the requirements.txt + # does not follow the Requirements File Format continue specs = req.specs @@ -153,7 +156,8 @@ def add_range(begin, end): add_range(begin, end) res.append( - {"forge": "PyPI", "product": req.name, "constraints": constraints} + {"forge": "PyPI", "product": req.name, + "constraints": constraints} ) return res @@ -183,8 +187,10 @@ def get_internal_modules(self): def add_superclasses(self, mods): for cls_name, cls in self.classes.items(): - cls_uri = self.namespace_map.get(self.to_uri(cls["module"], cls_name)) - mods[self.to_uri(cls["module"])]["namespaces"][cls_uri]["metadata"][ + cls_uri = self.namespace_map.get(self.to_uri(cls["module"], + cls_name)) + mods[self.to_uri(cls["module"])][ + "namespaces"][cls_uri]["metadata"][ "superClasses" ] = [] for parent in cls["mro"]: @@ -192,12 +198,14 @@ def add_superclasses(self, mods): continue if self.classes.get(parent): - parent_uri = self.to_uri(self.classes[parent]["module"], parent) + parent_uri = self.to_uri(self.classes[parent]["module"], + parent) else: parent_mod = parent.split(".")[0] parent_uri = self.to_external_uri(parent_mod, parent) - mods[self.to_uri(cls["module"])]["namespaces"][cls_uri]["metadata"][ + mods[self.to_uri(cls["module"])]["namespaces"][ + cls_uri]["metadata"][ "superClasses" ].append(parent_uri) @@ -205,7 +213,8 @@ def add_superclasses(self, mods): def create_namespaces_map(self): namespaces_maps = [{}, {}] - for res, hmap in zip(namespaces_maps, [self.internal_mods, self.external_mods]): + for res, hmap in zip(namespaces_maps, [self.internal_mods, + self.external_mods]): for mod in hmap: for namespace in hmap[mod]["methods"]: res[namespace] = mod @@ -246,13 +255,17 @@ def get_graph(self): uris.append(self.namespace_map.get(uri, uri)) elif node in external: mod = external[node] - uris.append(self.namespace_map.get(self.to_external_uri(mod, node))) + uris.append( + self.namespace_map.get(self.to_external_uri(mod, node)) + ) if len(uris) == 2: if dst in external: - graph["externalCalls"].append([str(uris[0]), str(uris[1]), {}]) + graph["externalCalls"].append([str(uris[0]), + str(uris[1]), {}]) else: - graph["internalCalls"].append([str(uris[0]), str(uris[1]), {}]) + graph["internalCalls"].append([str(uris[0]), + str(uris[1]), {}]) return graph def generate(self): diff --git a/pycg/machinery/classes.py b/pycg/machinery/classes.py index d9d46fe..dd2fd06 100644 --- a/pycg/machinery/classes.py +++ b/pycg/machinery/classes.py @@ -53,7 +53,7 @@ def add_parent(self, parent): def fix_mro(self): new_mro = [] for idx, item in enumerate(self.mro): - if self.mro[idx + 1 :].count(item) > 0: + if self.mro[idx + 1:].count(item) > 0: continue new_mro.append(item) self.mro = new_mro diff --git a/pycg/machinery/definitions.py b/pycg/machinery/definitions.py index 1002370..d2ec4f9 100644 --- a/pycg/machinery/definitions.py +++ b/pycg/machinery/definitions.py @@ -44,7 +44,8 @@ def assign(self, ns, defi): # if it is a function def, we need to create a return pointer if defi.is_function_def(): return_ns = utils.join_ns(ns, utils.constants.RETURN_NAME) - self.defs[return_ns] = Definition(return_ns, utils.constants.NAME_DEF) + self.defs[return_ns] = Definition(return_ns, + utils.constants.NAME_DEF) self.defs[return_ns].get_name_pointer().add( utils.join_ns(defi.get_ns(), utils.constants.RETURN_NAME) ) @@ -139,7 +140,8 @@ def update_pointsto_args(pointsto_args, arg, name): # if we remove this the following breaks: # x = lambda x: x + 1 # x(1) - # since on line 184 we don't discriminate between literal values and name values + # since on line 184 we don't discriminate between + # literal values and name values if not self.defs.get(item, None): continue pointsto_arg_def.add(item) @@ -160,22 +162,31 @@ def update_pointsto_args(pointsto_args, arg, name): continue pointsto_name_pointer = self.defs[name].get_name_pointer() - # iterate the arguments of the definition we're currently iterating - for arg_name, arg in current_name_pointer.get_args().items(): + # iterate the arguments of the definition + # we're currently iterating + for arg_name, arg in ( + current_name_pointer.get_args().items() + ): pos = current_name_pointer.get_pos_of_name(arg_name) if pos is not None: - pointsto_args = pointsto_name_pointer.get_pos_arg(pos) + pointsto_args = ( + pointsto_name_pointer.get_pos_arg(pos) + ) if not pointsto_args: - pointsto_name_pointer.add_pos_arg(pos, None, arg) + pointsto_name_pointer.add_pos_arg(pos, None, + arg) continue else: - pointsto_args = pointsto_name_pointer.get_arg(arg_name) + pointsto_args = ( + pointsto_name_pointer.get_arg(arg_name) + ) if not pointsto_args: pointsto_name_pointer.add_arg(arg_name, arg) continue - changed_something = changed_something or update_pointsto_args( - pointsto_args, arg, current_def.get_ns() - ) + changed_something = changed_something or \ + update_pointsto_args( + pointsto_args, arg, current_def.get_ns() + ) if not changed_something: break diff --git a/pycg/machinery/imports.py b/pycg/machinery/imports.py index c3d8b7a..b3159b4 100644 --- a/pycg/machinery/imports.py +++ b/pycg/machinery/imports.py @@ -135,8 +135,10 @@ def _handle_import_level(self, name, level): raise ImportError("Attempting import beyond top level package") mod_name = ("." * level) + name - # When an __init__ file is analyzed, then the module name doesn't contain - # the __init__ part in it, so special care must be taken for levels. + # When an __init__ file is analyzed, + # then the module name doesn't contain + # the __init__ part in it, + # so special care must be taken for levels. if self._is_init_file() and level >= 1: if level != 1: level -= 1 diff --git a/pycg/machinery/key_err.py b/pycg/machinery/key_err.py index 3b1a860..12fc042 100644 --- a/pycg/machinery/key_err.py +++ b/pycg/machinery/key_err.py @@ -24,7 +24,8 @@ def __init__(self): def add(self, filename, lineno, namespace, key): self.key_errs.append( - {"filename": filename, "lineno": lineno, "namespace": namespace, "key": key} + {"filename": filename, "lineno": lineno, "namespace": namespace, + "key": key} ) def get(self): diff --git a/pycg/machinery/scopes.py b/pycg/machinery/scopes.py index d4eb0f7..d8ba349 100644 --- a/pycg/machinery/scopes.py +++ b/pycg/machinery/scopes.py @@ -56,7 +56,8 @@ def process(namespace, parent, table): process(fullns, sc, t) process( - modulename, None, symtable.symtable(contents, filename, compile_type="exec") + modulename, None, symtable.symtable(contents, filename, + compile_type="exec") ) return {"functions": functions, "classes": classes} diff --git a/pycg/processing/base.py b/pycg/processing/base.py index 6ccb1a1..d283492 100644 --- a/pycg/processing/base.py +++ b/pycg/processing/base.py @@ -90,10 +90,12 @@ def visit_For(self, node): self.visit(item) def visit_Dict(self, node): - counter = self.scope_manager.get_scope(self.current_ns).inc_dict_counter() + counter = self.scope_manager.get_scope(self.current_ns). \ + inc_dict_counter() dict_name = utils.get_dict_name(counter) - sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, dict_name)) + sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, + dict_name)) if not sc: return self.name_stack.append(dict_name) @@ -106,10 +108,12 @@ def visit_Dict(self, node): self.name_stack.pop() def visit_List(self, node): - counter = self.scope_manager.get_scope(self.current_ns).inc_list_counter() + counter = self.scope_manager.get_scope(self.current_ns). \ + inc_list_counter() list_name = utils.get_list_name(counter) - sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, list_name)) + sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, + list_name)) if not sc: return self.name_stack.append(list_name) @@ -185,7 +189,8 @@ def do_assign(decoded, target): self.visit(target) if isinstance(target, ast.Tuple): for pos, elt in enumerate(target.elts): - if not isinstance(decoded, Definition) and pos < len(decoded): + if not isinstance(decoded, + Definition) and pos < len(decoded): do_assign(decoded[pos], elt) else: targetns = self._get_target_ns(target) @@ -283,7 +288,11 @@ def decode_node(self, node): return [] def _is_literal(self, item): - return isinstance(item, int) or isinstance(item, str) or isinstance(item, float) + return ( + isinstance(item, int) + or isinstance(item, str) + or isinstance(item, float) + ) def _retrieve_base_names(self, node): if not isinstance(node, ast.Attribute): @@ -357,7 +366,8 @@ def _retrieve_attribute_names(self, node): continue ext_name = utils.join_ns(name, node.attr) if not self.def_manager.get(ext_name): - self.def_manager.create(ext_name, utils.constants.EXT_DEF) + self.def_manager.create(ext_name, + utils.constants.EXT_DEF) names.add(ext_name) return names @@ -382,7 +392,8 @@ def iterate_call_args(self, defi, node): else: for d in decoded: if isinstance(d, Definition): - defi.get_name_pointer().add_pos_arg(pos, None, d.get_ns()) + defi.get_name_pointer().add_pos_arg(pos, None, + d.get_ns()) else: defi.get_name_pointer().add_pos_lit_arg(pos, None, d) @@ -405,7 +416,8 @@ def iterate_call_args(self, defi, node): else: for d in decoded: if isinstance(d, Definition): - defi.get_name_pointer().add_arg(keyword.arg, d.get_ns()) + defi.get_name_pointer().add_arg(keyword.arg, + d.get_ns()) else: defi.get_name_pointer().add_lit_arg(keyword.arg, d) @@ -416,7 +428,8 @@ def retrieve_subscript_names(self, node): if not getattr(self, "closured", None): return set() - if getattr(node.slice, "value", None) and self._is_literal(node.slice.value): + if getattr(node.slice, "value", None) and \ + self._is_literal(node.slice.value): sl_names = [node.slice.value] else: sl_names = self.decode_node(node.slice) @@ -428,10 +441,12 @@ def retrieve_subscript_names(self, node): full_names = set() # get all names associated with this variable name for n in val_names: - if n and isinstance(n, Definition) and self.closured.get(n.get_ns(), None): + if n and isinstance(n, Definition) and \ + self.closured.get(n.get_ns(), None): decoded_vals |= self.closured.get(n.get_ns()) for s in sl_names: - if isinstance(s, Definition) and self.closured.get(s.get_ns(), None): + if isinstance(s, Definition) and \ + self.closured.get(s.get_ns(), None): # we care about the literals pointed by the name # not the namespaces, so retrieve the literals pointed for name in self.closured.get(s.get_ns()): @@ -538,7 +553,8 @@ def add_ext_mod_node(self, name): ext_modname = name.split(".")[0] ext_mod = self.module_manager.get(ext_modname) if not ext_mod: - ext_mod = self.module_manager.create(ext_modname, None, external=True) + ext_mod = self.module_manager.create(ext_modname, None, + external=True) ext_mod.add_method(ext_modname) ext_mod.add_method(name) diff --git a/pycg/processing/cgprocessor.py b/pycg/processing/cgprocessor.py index 752f3de..9f91be3 100644 --- a/pycg/processing/cgprocessor.py +++ b/pycg/processing/cgprocessor.py @@ -78,7 +78,8 @@ def visit_For(self, node): super().visit_For(node) def visit_Lambda(self, node): - counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() + counter = self.scope_manager.get_scope(self.current_ns).\ + inc_lambda_counter() lambda_name = utils.get_lambda_name(counter) lambda_fullns = utils.join_ns(self.current_ns, lambda_name) @@ -98,7 +99,8 @@ def visit_Raise(self, node): for name in names: pointer_def = self.def_manager.get(name) if pointer_def.get_type() == utils.constants.CLS_DEF: - init_ns = self.find_cls_fun_ns(name, utils.constants.CLS_INIT) + init_ns = self.find_cls_fun_ns(name, + utils.constants.CLS_INIT) for ns in init_ns: self.call_graph.add_edge(self.current_method, ns) if pointer_def.get_type() == utils.constants.EXT_DEF: @@ -142,14 +144,18 @@ def create_ext_edge(name, ext_modname): names = self.retrieve_call_names(node) if not names: - if isinstance(node.func, ast.Attribute) and self.has_ext_parent(node.func): - # TODO: This doesn't work for cases where there is an assignment of an attribute + if isinstance(node.func, ast.Attribute) and \ + self.has_ext_parent(node.func): + # TODO: This doesn't work for cases + # where there is an assignment of an attribute # i.e. import os; lala = os.path; lala.dirname() for name in self.get_full_attr_names(node.func): ext_modname = name.split(".")[0] create_ext_edge(name, ext_modname) - elif getattr(node.func, "id", None) and self.is_builtin(node.func.id): - name = utils.join_ns(utils.constants.BUILTIN_NAME, node.func.id) + elif getattr(node.func, "id", None) and \ + self.is_builtin(node.func.id): + name = utils.join_ns(utils.constants.BUILTIN_NAME, + node.func.id) create_ext_edge(name, utils.constants.BUILTIN_NAME) return @@ -165,16 +171,20 @@ def create_ext_edge(name, ext_modname): continue self.call_graph.add_edge(self.current_method, pointer) - # TODO: This doesn't work and leads to calls from the decorators - # themselves to the function, creating edges to the first decorator - # for decorator in pointer_def.decorator_names: - # dec_names = self.closured.get(decorator, []) - # for dec_name in dec_names: - # if self.def_manager.get(dec_name).get_type() == utils.constants.FUN_DEF: - # self.call_graph.add_edge(self.current_ns, dec_name) + # TODO: This doesn't work + # and leads to calls from the decorators + # themselves to the function, + # creating edges to the first decorator + # for decorator in pointer_def.decorator_names: + # dec_names = self.closured.get(decorator, []) + # for dec_name in dec_names: + # if self.def_manager.get(dec_name). + # get_type() == utils.constants.FUN_DEF: + # self.call_graph.add_edge(self.current_ns, dec_name) if pointer_def.get_type() == utils.constants.CLS_DEF: - init_ns = self.find_cls_fun_ns(pointer, utils.constants.CLS_INIT) + init_ns = self.find_cls_fun_ns(pointer, + utils.constants.CLS_INIT) for ns in init_ns: self.call_graph.add_edge(self.current_method, ns) diff --git a/pycg/processing/keyerrprocessor.py b/pycg/processing/keyerrprocessor.py index c469215..d8bcbcb 100644 --- a/pycg/processing/keyerrprocessor.py +++ b/pycg/processing/keyerrprocessor.py @@ -94,7 +94,8 @@ def analyze(self): self.analyze_submodules() def visit_Lambda(self, node): - counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() + counter = self.scope_manager.get_scope(self.current_ns).\ + inc_lambda_counter() lambda_name = utils.get_lambda_name(counter) utils.join_ns(self.current_ns, lambda_name) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index dcb5e67..f04d6d0 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -46,7 +46,9 @@ def __init__( self.closured = self.def_manager.transitive_closure() def visit_Lambda(self, node): - counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() + counter = self.scope_manager.\ + get_scope(self.current_ns).\ + inc_lambda_counter() lambda_name = utils.get_lambda_name(counter) super().visit_Lambda(node, lambda_name) @@ -108,9 +110,11 @@ def visit_For(self, node): ) ) if next_defi: - for name in self.closured.get(next_defi.get_ns(), []): + for name in self.closured.get(next_defi.get_ns(), + []): target_def.get_name_pointer().add(name) - else: # otherwise, add a pointer to the name (e.g. a yield) + else: # otherwise, add a pointer to the name + # (e.g. a yield) target_def.get_name_pointer().add(name) super().visit_For(node) @@ -127,7 +131,8 @@ def visit_AsyncFunctionDef(self, node): def visit_FunctionDef(self, node): # here we iterate decorators if node.decorator_list: - fn_def = self.def_manager.get(utils.join_ns(self.current_ns, node.name)) + fn_def = self.def_manager.get(utils.join_ns(self.current_ns, + node.name)) reversed_decorators = list(reversed(node.decorator_list)) # add to the name pointer of the function definition @@ -144,14 +149,16 @@ def visit_FunctionDef(self, node): previous_names = self.closured.get(fn_def.get_ns(), set()) for decorator in reversed_decorators: - # assign the previous_def as the first parameter of the decorator + # assign the previous_def + # as the first parameter of the decorator decoded = self.decode_node(decorator) new_previous_names = set() for d in decoded: if not isinstance(d, Definition): continue for name in self.closured.get(d.get_ns(), []): - return_ns = utils.join_ns(name, utils.constants.RETURN_NAME) + return_ns = utils.join_ns(name, + utils.constants.RETURN_NAME) if self.closured.get(return_ns, None) is None: continue @@ -220,16 +227,19 @@ def visit_List(self, node): # create a list definition list_def = self.def_manager.get(list_full_ns) if not list_def: - list_def = self.def_manager.create(list_full_ns, utils.constants.NAME_DEF) + list_def = self.def_manager.create(list_full_ns, + utils.constants.NAME_DEF) current_scope.add_def(list_name, list_def) self.name_stack.append(list_name) for idx, elt in enumerate(node.elts): self.visit(elt) - key_full_ns = utils.join_ns(list_def.get_ns(), utils.get_int_name(idx)) + key_full_ns = utils.join_ns(list_def.get_ns(), + utils.get_int_name(idx)) key_def = self.def_manager.get(key_full_ns) if not key_def: - key_def = self.def_manager.create(key_full_ns, utils.constants.NAME_DEF) + key_def = self.def_manager.create(key_full_ns, + utils.constants.NAME_DEF) decoded_elt = self.decode_node(elt) for v in decoded_elt: @@ -250,12 +260,14 @@ def visit_Dict(self, node): dict_full_ns = utils.join_ns(self.current_ns, dict_name) # create a scope for the dict - dict_scope = self.scope_manager.create_scope(dict_full_ns, current_scope) + dict_scope = self.scope_manager.create_scope(dict_full_ns, + current_scope) # Create a dict definition dict_def = self.def_manager.get(dict_full_ns) if not dict_def: - dict_def = self.def_manager.create(dict_full_ns, utils.constants.NAME_DEF) + dict_def = self.def_manager.create(dict_full_ns, + utils.constants.NAME_DEF) # add it to the current scope current_scope.add_def(dict_name, dict_def) @@ -319,7 +331,8 @@ def update_parent_classes(self, defi): new_ns = utils.join_ns(parent_def.get_ns(), key) new_def = self.def_manager.get(new_ns) if not new_def: - new_def = self.def_manager.create(new_ns, utils.constants.NAME_DEF) + new_def = self.def_manager.create(new_ns, + utils.constants.NAME_DEF) new_def.get_name_pointer().add_set(names) new_def.get_name_pointer().add(child_def.get_ns()) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 84e9ed4..08bfb7d 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -125,7 +125,8 @@ def iterate_mod_items(items, const): defi = self.def_manager.get(self.modname) if not defi: - defi = self.def_manager.create(self.modname, utils.constants.MOD_DEF) + defi = self.def_manager.create(self.modname, + utils.constants.MOD_DEF) super().visit_Module(node) @@ -162,7 +163,9 @@ def create_def(scope, name, imported_def): if tgt_name == "*": for name, defi in imported_scope.get_defs().items(): create_def(current_scope, name, defi) - current_scope.get_def(name).get_name_pointer().add(defi.get_ns()) + current_scope.get_def(name).get_name_pointer().add( + defi.get_ns() + ) else: # if it exists in the imported scope then copy it defi = imported_scope.get_def(imp_name) @@ -187,13 +190,17 @@ def add_external_def(name, target): tgt_ns = utils.join_ns(scope.get_ns(), target) tgt_defi = self.def_manager.get(tgt_ns) if not tgt_defi: - tgt_defi = self.def_manager.create(tgt_ns, utils.constants.EXT_DEF) + tgt_defi = self.def_manager.create( + tgt_ns, utils.constants.EXT_DEF + ) tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) for import_item in node.names: src_name = handle_src_name(import_item.name) - tgt_name = import_item.asname if import_item.asname else import_item.name + tgt_name = ( + import_item.asname if import_item.asname else import_item.name + ) imported_name = self.import_manager.handle_import(src_name, level) if not imported_name: @@ -258,7 +265,8 @@ def _handle_function_def(self, node, fn_name): defs_to_create = [] name_pointer = fn_def.get_name_pointer() - # TODO: static methods can be created using the staticmethod() function too + # TODO: static methods can be created using + # the staticmethod() function too is_static_method = False if hasattr(node, "decorator_list"): for decorator in node.decorator_list: @@ -276,7 +284,8 @@ def _handle_function_def(self, node, fn_name): arg_ns = utils.join_ns(fn_def.get_ns(), node.args.args[0].arg) arg_def = self.def_manager.get(arg_ns) if not arg_def: - arg_def = self.def_manager.create(arg_ns, utils.constants.NAME_DEF) + arg_def = self.def_manager.create(arg_ns, + utils.constants.NAME_DEF) arg_def.get_name_pointer().add(current_def.get_ns()) self.scope_manager.handle_assign( @@ -304,7 +313,8 @@ def _handle_function_def(self, node, fn_name): for arg_ns in defs_to_create: arg_def = self.def_manager.get(arg_ns) if not arg_def: - arg_def = self.def_manager.create(arg_ns, utils.constants.NAME_DEF) + arg_def = self.def_manager.create(arg_ns, + utils.constants.NAME_DEF) self.scope_manager.handle_assign( fn_def.get_ns(), arg_def.get_name(), arg_def @@ -337,7 +347,8 @@ def visit_For(self, node): if isinstance(node.target, ast.Name): target_ns = utils.join_ns(self.current_ns, node.target.id) if not self.def_manager.get(target_ns): - defi = self.def_manager.create(target_ns, utils.constants.NAME_DEF) + defi = self.def_manager.create(target_ns, + utils.constants.NAME_DEF) self.scope_manager.get_scope(self.current_ns).add_def( node.target.id, defi ) @@ -397,7 +408,8 @@ def visit_ClassDef(self, node): mod = self.module_manager.get(self.modname) if not mod: mod = self.module_manager.create(self.modname, self.filename) - mod.add_method(cls_def.get_ns(), node.lineno, self._get_last_line(node)) + mod.add_method(cls_def.get_ns(), node.lineno, + self._get_last_line(node)) # iterate bases to compute MRO for the class cls = self.class_manager.get(cls_def.get_ns()) diff --git a/pycg/tests/definitions_test.py b/pycg/tests/definitions_test.py index 6c3975c..94968bd 100644 --- a/pycg/tests/definitions_test.py +++ b/pycg/tests/definitions_test.py @@ -61,13 +61,15 @@ def test_assign(self): self.assertEqual(defi2.get_ns(), "defi2") # values should be merged self.assertEqual(defi2.get_type(), utils.constants.NAME_DEF) - self.assertEqual(defi2.get_name_pointer().get(), set(["item1", "item2"])) + self.assertEqual(defi2.get_name_pointer().get(), + set(["item1", "item2"])) self.assertEqual(defi2.get_name_pointer().get_arg(0), set(["arg"])) # for function defs a return def should be created too fndefi = dm.create("fndefi", utils.constants.FUN_DEF) dm.assign("fndefi2", fndefi) - return_def = dm.get("{}.{}".format("fndefi2", utils.constants.RETURN_NAME)) + return_def = dm.get("{}.{}".format("fndefi2", + utils.constants.RETURN_NAME)) self.assertIsNotNone(return_def) self.assertEqual( return_def.get_name_pointer().get(), diff --git a/pycg/tests/fasten_format_test.py b/pycg/tests/fasten_format_test.py index f88ca71..1e3dcca 100644 --- a/pycg/tests/fasten_format_test.py +++ b/pycg/tests/fasten_format_test.py @@ -90,7 +90,7 @@ def test_uri(self): self.cg_generator.functions = ["mod1.mod2.myfunc"] formatter = self.get_formatter() - ### Internal uri check + # Internal uri check # test modname without method self.assertEqual(formatter.to_uri("mymod"), "/mymod/") self.assertEqual(formatter.to_uri("mymod.mod1"), "/mymod.mod1/") @@ -99,17 +99,19 @@ def test_uri(self): formatter.to_uri("mymod.mod1", "mymod.mod1.fn"), "/mymod.mod1/fn" ) self.assertEqual( - formatter.to_uri("mymod.mod1", "mymod.mod1.cls.fn"), "/mymod.mod1/cls.fn" + formatter.to_uri("mymod.mod1", "mymod.mod1.cls.fn"), + "/mymod.mod1/cls.fn" ) # test method starting with modname but without . inbetween with self.assertRaises(Exception): formatter.to_uri("mymod.mod1", "mymod.mod1cls.fn") # test method being in functions self.assertEqual( - formatter.to_uri("mod1.mod2", "mod1.mod2.myfunc"), "/mod1.mod2/myfunc()" + formatter.to_uri("mod1.mod2", "mod1.mod2.myfunc"), + "/mod1.mod2/myfunc()" ) - ### External uri check + # External uri check # test modname builtin self.assertEqual( formatter.to_external_uri( @@ -129,7 +131,8 @@ def _get_internal_mods(self): "mod1": { "filename": "mod1.py", "methods": { - "mod1.method": {"name": "mod1.method", "first": 2, "last": 5}, + "mod1.method": {"name": "mod1.method", "first": 2, + "last": 5}, "mod1.Cls.method": { "name": "mod1.Cls.method", "first": 6, @@ -153,7 +156,8 @@ def _get_internal_mods(self): "first": 6, "last": 9, }, - "mod.mod2.Cls": {"name": "mod.mod2.Cls", "first": 4, "last": 9}, + "mod.mod2.Cls": {"name": "mod.mod2.Cls", "first": 4, + "last": 9}, "mod.mod2.Cls.Nested": { "name": "mod.mod2.Cls.Nested", "first": 5, @@ -168,7 +172,8 @@ def _get_external_mods(self): "external": { "filename": None, "methods": { - "external": {"name": "external", "first": None, "last": None}, + "external": {"name": "external", "first": None, + "last": None}, "external.Cls": { "name": "external.Cls", "first": None, @@ -184,7 +189,8 @@ def _get_external_mods(self): "external2": { "filename": None, "methods": { - "external2": {"name": "external2", "first": None, "last": None}, + "external2": {"name": "external2", "first": None, + "last": None}, "external2.method": { "name": "external2.method", "first": None, @@ -197,7 +203,8 @@ def _get_external_mods(self): def _get_classes(self): return { "mod1.Cls": {"module": "mod1", "mro": ["mod1.Cls"]}, - "mod.mod2.Cls": {"module": "mod.mod2", "mro": ["mod.mod2.Cls", "mod1.Cls"]}, + "mod.mod2.Cls": {"module": "mod.mod2", "mro": ["mod.mod2.Cls", + "mod1.Cls"]}, "mod.mod2.Cls.Nested": { "module": "mod.mod2", "mro": ["mod.mod2.Cls.Nested", "external.Cls"], @@ -219,7 +226,8 @@ def test_internal_modules(self): # test that SourceFileName are correct for name, mod in internal_mods.items(): self.assertEqual( - mod["filename"], internal_modules[formatter.to_uri(name)]["sourceFile"] + mod["filename"], + internal_modules[formatter.to_uri(name)]["sourceFile"] ) # test that namespaces contains all methods @@ -233,11 +241,15 @@ def test_internal_modules(self): first = info["first"] last = info["last"] expected_namespaces.append( - dict(namespace=method_uri, metadata=dict(first=first, last=last)) + dict(namespace=method_uri, metadata=dict(first=first, + last=last)) ) # namespaces defined for module - result_namespaces = internal_modules[name_uri]["namespaces"].values() + result_namespaces = ( + internal_modules[name_uri]["namespaces"]. + values() + ) # unique identifiers defined for module result_ids = internal_modules[name_uri]["namespaces"].keys() @@ -256,8 +268,10 @@ def test_external_modules(self): external_modules = formatter.generate()["modules"]["external"] # test that external modules keys are identical with the ones generated - self.assertEqual(set(external_mods.keys()), set(external_modules.keys())) - self.assertEqual(len(external_mods.keys()), len(external_modules.keys())) + self.assertEqual(set(external_mods.keys()), + set(external_modules.keys())) + self.assertEqual(len(external_mods.keys()), + len(external_modules.keys())) # test that namespaces contains all the expected methods for name, mod in external_mods.items(): @@ -266,7 +280,8 @@ def test_external_modules(self): for method, info in mod["methods"].items(): if method != name: method_uri = formatter.to_external_uri(name, method) - expected_namespaces.append(dict(namespace=method_uri, metadata={})) + expected_namespaces.append(dict(namespace=method_uri, + metadata={})) # namespaces defined for external modules result_namespaces = external_modules[name]["namespaces"].values() @@ -299,7 +314,8 @@ def test_hiearchy(self): total_classes += 1 self.assertEqual(total_classes, len(classes.keys())) for cls_name, cls in classes.items(): - cls_name_uri = id_mapping[formatter.to_uri(cls["module"], cls_name)] + cls_name_uri = id_mapping[formatter.to_uri(cls["module"], + cls_name)] cls_mro = [] for item in cls["mro"]: # result mro should not contain the class name @@ -307,13 +323,17 @@ def test_hiearchy(self): continue if classes.get(item, None): # it is an internal module - cls_mro.append(formatter.to_uri(classes[item]["module"], item)) + cls_mro.append(formatter.to_uri(classes[item]["module"], + item)) else: - cls_mro.append(formatter.to_external_uri(item.split(".")[0], item)) + cls_mro.append( + formatter.to_external_uri(item.split(".")[0], + item)) self.assertEqual( cls_mro, - modules[formatter.to_uri(cls["module"])]["namespaces"][cls_name_uri][ + modules[formatter.to_uri(cls["module"])]["namespaces"] + [cls_name_uri][ "metadata" ]["superClasses"], ) diff --git a/pycg/tests/imports_test.py b/pycg/tests/imports_test.py index 9209841..6b02b7e 100644 --- a/pycg/tests/imports_test.py +++ b/pycg/tests/imports_test.py @@ -25,7 +25,9 @@ import mock from base import TestBase -from pycg.machinery.imports import ImportManager, ImportManagerError, get_custom_loader +from pycg.machinery.imports import ( + ImportManager, ImportManagerError, get_custom_loader +) class ImportsTest(TestBase): @@ -106,12 +108,14 @@ def test_hooks(self): custom_loader = "custom_loader" with mock.patch( - "importlib.machinery.FileFinder.path_hook", return_value=custom_loader + "importlib.machinery.FileFinder.path_hook", + return_value=custom_loader ): im.install_hooks() self.assertEqual(sys.path_hooks[0], custom_loader) - self.assertEqual(sys.path[0], os.path.abspath(os.path.dirname(input_file))) + self.assertEqual(sys.path[0], + os.path.abspath(os.path.dirname(input_file))) im.remove_hooks() self.assertEqual(old_sys_path, sys.path) @@ -149,9 +153,12 @@ def test_handle_import_level(self): with self.assertRaises(ImportError): im._handle_import_level("something", 4) - self.assertEqual(im._handle_import_level("smth", 2), ("..smth", "mod1")) - self.assertEqual(im._handle_import_level("smth", 1), (".smth", "mod1.mod2")) - self.assertEqual(im._handle_import_level("smth", 0), ("smth", "")) + self.assertEqual(im._handle_import_level("smth", 2), + ("..smth", "mod1")) + self.assertEqual(im._handle_import_level("smth", 1), + (".smth", "mod1.mod2")) + self.assertEqual(im._handle_import_level("smth", 0), + ("smth", "")) def test_handle_import(self): # test builtin modules diff --git a/pycg/tests/pointers_test.py b/pycg/tests/pointers_test.py index 4eef454..8a27287 100644 --- a/pycg/tests/pointers_test.py +++ b/pycg/tests/pointers_test.py @@ -20,7 +20,9 @@ # from base import TestBase -from pycg.machinery.pointers import LiteralPointer, NamePointer, Pointer, PointerError +from pycg.machinery.pointers import ( + LiteralPointer, NamePointer, Pointer, PointerError +) class PointerTest(TestBase): @@ -108,5 +110,6 @@ def test_name_pointer_merge(self): pointer1.merge(pointer2) self.assertEqual(pointer1.get(), set(["smth1", "smth6"])) - self.assertEqual(pointer1.get_arg(0), set(["smth2", "smth3", "smth7", "smth8"])) + self.assertEqual(pointer1.get_arg(0), + set(["smth2", "smth3", "smth7", "smth8"])) self.assertEqual(pointer1.get_arg(1), set(["smth4", "smth9"])) diff --git a/pycg/tests/scopes_test.py b/pycg/tests/scopes_test.py index c14d67b..4ea89a6 100644 --- a/pycg/tests/scopes_test.py +++ b/pycg/tests/scopes_test.py @@ -77,33 +77,42 @@ def get_lineno(self): self.assertEqual(sm.get_scope("root").parent, None) self.assertEqual(sm.get_scope("root.chld1").get_ns(), "root.chld1") - self.assertEqual(sm.get_scope("root.chld1").parent, sm.get_scope("root")) + self.assertEqual(sm.get_scope("root.chld1").parent, + sm.get_scope("root")) self.assertEqual(sm.get_scope("root.chld2").get_ns(), "root.chld2") - self.assertEqual(sm.get_scope("root.chld2").parent, sm.get_scope("root")) + self.assertEqual(sm.get_scope("root.chld2").parent, + sm.get_scope("root")) self.assertEqual(sm.get_scope("root.chld3").get_ns(), "root.chld3") - self.assertEqual(sm.get_scope("root.chld3").parent, sm.get_scope("root")) + self.assertEqual(sm.get_scope("root.chld3").parent, + sm.get_scope("root")) self.assertEqual( - sm.get_scope("root.chld1.grndchld1").get_ns(), "root.chld1.grndchld1" + sm.get_scope("root.chld1.grndchld1").get_ns(), + "root.chld1.grndchld1" ) self.assertEqual( - sm.get_scope("root.chld1.grndchld1").parent, sm.get_scope("root.chld1") + sm.get_scope("root.chld1.grndchld1").parent, + sm.get_scope("root.chld1") ) self.assertEqual( - sm.get_scope("root.chld1.grndchld2").get_ns(), "root.chld1.grndchld2" + sm.get_scope("root.chld1.grndchld2").get_ns(), + "root.chld1.grndchld2" ) self.assertEqual( - sm.get_scope("root.chld1.grndchld2").parent, sm.get_scope("root.chld1") + sm.get_scope("root.chld1.grndchld2").parent, + sm.get_scope("root.chld1") ) self.assertEqual( - sm.get_scope("root.chld2.grndchld3").get_ns(), "root.chld2.grndchld3" + sm.get_scope("root.chld2.grndchld3").get_ns(), + "root.chld2.grndchld3" ) self.assertEqual( - sm.get_scope("root.chld2.grndchld3").parent, sm.get_scope("root.chld2") + sm.get_scope("root.chld2.grndchld3").parent, + sm.get_scope("root.chld2") ) def test_handle_assign(self): @@ -122,7 +131,8 @@ def test_get_def(self): sm.scopes[root] = ScopeItem(root, None) # root scope sm.scopes[chld1] = ScopeItem(chld1, sm.scopes[root]) # 1st child scope sm.scopes[chld2] = ScopeItem(chld2, sm.scopes[root]) # 2nd child scope - sm.scopes[grndchld] = ScopeItem(grndchld, sm.scopes[chld1]) # grandchild + sm.scopes[grndchld] = ScopeItem(grndchld, + sm.scopes[chld1]) # grandchild grndchld_def = ("var", "grndchild_def") # name, value chld1_def1 = ("var", "chld1_def") @@ -138,15 +148,20 @@ def test_get_def(self): # should be able to find a variable defined in its scope # also it should get the value of the nearest scope, meaning it's own - self.assertEqual(sm.get_def("ns.chld1.chld1", grndchld_def[0]), grndchld_def[1]) + self.assertEqual(sm.get_def("ns.chld1.chld1", + grndchld_def[0]), grndchld_def[1]) # if it doesn't exist get the parent's - self.assertEqual(sm.get_def("ns.chld1.chld1", chld1_def2[0]), chld1_def2[1]) + self.assertEqual(sm.get_def("ns.chld1.chld1", + chld1_def2[0]), chld1_def2[1]) # it shouldn't be able to reach a def defined in chld2 - self.assertEqual(sm.get_def("ns.chld1.chld1", chld2_def[0]), None) + self.assertEqual(sm.get_def("ns.chld1.chld1", + chld2_def[0]), None) # root doesn't have access to variables defined in lower scopes - self.assertEqual(sm.get_def("ns", chld1_def2[0]), None) + self.assertEqual(sm.get_def("ns", + chld1_def2[0]), None) # but childs have access to root - self.assertEqual(sm.get_def("ns.chld2", root_def[0]), root_def[1]) + self.assertEqual(sm.get_def("ns.chld2", + root_def[0]), root_def[1]) def test_get_scope(self): sm = ScopeManager() From 7759c4de07e2b5118949b8cb1712968c9dbbc4bd Mon Sep 17 00:00:00 2001 From: Giorgos Drosos Date: Tue, 16 May 2023 14:58:39 +0300 Subject: [PATCH 4/7] Set max line length to 89 --- .github/workflows/linters.yml | 2 +- pycg/processing/cgprocessor.py | 20 ++++++++++---------- pycg/tests/imports_test.py | 4 +--- pycg/tests/pointers_test.py | 4 +--- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 29a427c..e8290cd 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -31,7 +31,7 @@ jobs: uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2 with: linters: flake8 - run: flake8 pycg + run: flake8 --max-line-length 89 pycg isort: runs-on: ubuntu-latest diff --git a/pycg/processing/cgprocessor.py b/pycg/processing/cgprocessor.py index 9f91be3..855d6a3 100644 --- a/pycg/processing/cgprocessor.py +++ b/pycg/processing/cgprocessor.py @@ -171,16 +171,16 @@ def create_ext_edge(name, ext_modname): continue self.call_graph.add_edge(self.current_method, pointer) - # TODO: This doesn't work - # and leads to calls from the decorators - # themselves to the function, - # creating edges to the first decorator - # for decorator in pointer_def.decorator_names: - # dec_names = self.closured.get(decorator, []) - # for dec_name in dec_names: - # if self.def_manager.get(dec_name). - # get_type() == utils.constants.FUN_DEF: - # self.call_graph.add_edge(self.current_ns, dec_name) + # TODO: This doesn't work + # and leads to calls from the decorators + # themselves to the function, + # creating edges to the first decorator + # for decorator in pointer_def.decorator_names: + # dec_names = self.closured.get(decorator, []) + # for dec_name in dec_names: + # if self.def_manager.get(dec_name). + # get_type() == utils.constants.FUN_DEF: + # self.call_graph.add_edge(self.current_ns, dec_name) if pointer_def.get_type() == utils.constants.CLS_DEF: init_ns = self.find_cls_fun_ns(pointer, diff --git a/pycg/tests/imports_test.py b/pycg/tests/imports_test.py index 6b02b7e..0d495ab 100644 --- a/pycg/tests/imports_test.py +++ b/pycg/tests/imports_test.py @@ -25,9 +25,7 @@ import mock from base import TestBase -from pycg.machinery.imports import ( - ImportManager, ImportManagerError, get_custom_loader -) +from pycg.machinery.imports import ImportManager, ImportManagerError, get_custom_loader class ImportsTest(TestBase): diff --git a/pycg/tests/pointers_test.py b/pycg/tests/pointers_test.py index 8a27287..c0e143b 100644 --- a/pycg/tests/pointers_test.py +++ b/pycg/tests/pointers_test.py @@ -20,9 +20,7 @@ # from base import TestBase -from pycg.machinery.pointers import ( - LiteralPointer, NamePointer, Pointer, PointerError -) +from pycg.machinery.pointers import LiteralPointer, NamePointer, Pointer, PointerError class PointerTest(TestBase): From 5617940e9f885d80a474ee9024ab3e7983b94012 Mon Sep 17 00:00:00 2001 From: Giorgos Drosos Date: Tue, 16 May 2023 15:05:11 +0300 Subject: [PATCH 5/7] Resolve final flake8 linting issues in imports --- pycg/formats/__init__.py | 6 +++--- pycg/utils/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pycg/formats/__init__.py b/pycg/formats/__init__.py index 813810a..ad3775a 100644 --- a/pycg/formats/__init__.py +++ b/pycg/formats/__init__.py @@ -18,6 +18,6 @@ # specific language governing permissions and limitations # under the License. # -from .as_graph import AsGraph -from .fasten import Fasten -from .simple import Simple +from .as_graph import AsGraph # noqa: F401 +from .fasten import Fasten # noqa: F401 +from .simple import Simple # noqa: F401 diff --git a/pycg/utils/__init__.py b/pycg/utils/__init__.py index 9091903..b873773 100644 --- a/pycg/utils/__init__.py +++ b/pycg/utils/__init__.py @@ -18,5 +18,5 @@ # specific language governing permissions and limitations # under the License. # -from . import constants +from . import constants # noqa: F401 from .common import * # noqa From b3c9ddf1327ee38f1936d6deedd21fc9727036fa Mon Sep 17 00:00:00 2001 From: Giorgos Drosos Date: Tue, 16 May 2023 15:10:51 +0300 Subject: [PATCH 6/7] Reformat files based on black --- pycg/__main__.py | 28 +++++----------- pycg/formats/base.py | 4 +-- pycg/formats/fasten.py | 38 +++++++-------------- pycg/machinery/classes.py | 2 +- pycg/machinery/definitions.py | 25 +++++--------- pycg/machinery/key_err.py | 3 +- pycg/machinery/scopes.py | 3 +- pycg/processing/base.py | 42 +++++++---------------- pycg/processing/cgprocessor.py | 18 ++++------ pycg/processing/keyerrprocessor.py | 3 +- pycg/processing/postprocessor.py | 31 ++++++----------- pycg/processing/preprocessor.py | 27 +++++---------- pycg/tests/definitions_test.py | 6 ++-- pycg/tests/fasten_format_test.py | 54 ++++++++++-------------------- pycg/tests/imports_test.py | 15 +++------ pycg/tests/pointers_test.py | 3 +- pycg/tests/scopes_test.py | 45 +++++++++---------------- setup.py | 31 +++++++++-------- 18 files changed, 129 insertions(+), 249 deletions(-) diff --git a/pycg/__main__.py b/pycg/__main__.py index 3b5517c..a109308 100644 --- a/pycg/__main__.py +++ b/pycg/__main__.py @@ -8,12 +8,9 @@ def main(): parser = argparse.ArgumentParser() - parser.add_argument("entry_point", - nargs="*", - help="Entry points to be processed") + parser.add_argument("entry_point", nargs="*", help="Entry points to be processed") parser.add_argument( - "--package", - help="Package containing the code to be analyzed", default=None + "--package", help="Package containing the code to be analyzed", default=None ) parser.add_argument( "--fasten", @@ -21,17 +18,13 @@ def main(): action="store_true", default=False, ) - parser.add_argument("--product", - help="Package name", default="") + parser.add_argument("--product", help="Package name", default="") parser.add_argument( - "--forge", - help="Source the product was downloaded from", default="" + "--forge", help="Source the product was downloaded from", default="" ) - parser.add_argument("--version", - help="Version of the product", default="") + parser.add_argument("--version", help="Version of the product", default="") parser.add_argument( - "--timestamp", - help="Timestamp of the package's version", default=0 + "--timestamp", help="Timestamp of the package's version", default=0 ) parser.add_argument( "--max-iter", @@ -59,11 +52,9 @@ def main(): ) parser.add_argument( - "--as-graph-output", - help="Output for the assignment graph", default=None + "--as-graph-output", help="Output for the assignment graph", default=None ) - parser.add_argument("-o", "--output", - help="Output path", default=None) + parser.add_argument("-o", "--output", help="Output path", default=None) args = parser.parse_args() @@ -75,8 +66,7 @@ def main(): if args.operation == CALL_GRAPH_OP: if args.fasten: formatter = formats.Fasten( - cg, args.package, args.product, - args.forge, args.version, args.timestamp + cg, args.package, args.product, args.forge, args.version, args.timestamp ) else: formatter = formats.Simple(cg) diff --git a/pycg/formats/base.py b/pycg/formats/base.py index cb4c0cc..88f13bd 100644 --- a/pycg/formats/base.py +++ b/pycg/formats/base.py @@ -20,6 +20,4 @@ # class BaseFormatter: def generate(self): - raise NotImplementedError( - "generate method not implemented in child class" - ) + raise NotImplementedError("generate method not implemented in child class") diff --git a/pycg/formats/fasten.py b/pycg/formats/fasten.py index adfbcbf..3d56f56 100644 --- a/pycg/formats/fasten.py +++ b/pycg/formats/fasten.py @@ -28,8 +28,7 @@ class Fasten(BaseFormatter): - def __init__(self, cg_generator, package, product, forge, version, - timestamp): + def __init__(self, cg_generator, package, product, forge, version, timestamp): self.cg_generator = cg_generator self.internal_mods = self.cg_generator.output_internal_mods() or {} self.external_mods = self.cg_generator.output_external_mods() or {} @@ -56,10 +55,9 @@ def to_uri(self, modname, name=""): cleared = "" else: if not name.startswith(modname + "."): - raise Exception("name should start with modname", name, - modname) + raise Exception("name should start with modname", name, modname) - cleared = name[len(modname) + 1:] + cleared = name[len(modname) + 1 :] suffix = "" if name in self.functions: @@ -69,7 +67,7 @@ def to_uri(self, modname, name=""): def to_external_uri(self, modname, name=""): if modname == utils.constants.BUILTIN_NAME: - name = name[len(modname) + 1:] + name = name[len(modname) + 1 :] modname = ".builtin" return "//{}//{}".format(modname.replace("-", "_"), name) @@ -156,8 +154,7 @@ def add_range(begin, end): add_range(begin, end) res.append( - {"forge": "PyPI", "product": req.name, - "constraints": constraints} + {"forge": "PyPI", "product": req.name, "constraints": constraints} ) return res @@ -187,10 +184,8 @@ def get_internal_modules(self): def add_superclasses(self, mods): for cls_name, cls in self.classes.items(): - cls_uri = self.namespace_map.get(self.to_uri(cls["module"], - cls_name)) - mods[self.to_uri(cls["module"])][ - "namespaces"][cls_uri]["metadata"][ + cls_uri = self.namespace_map.get(self.to_uri(cls["module"], cls_name)) + mods[self.to_uri(cls["module"])]["namespaces"][cls_uri]["metadata"][ "superClasses" ] = [] for parent in cls["mro"]: @@ -198,14 +193,12 @@ def add_superclasses(self, mods): continue if self.classes.get(parent): - parent_uri = self.to_uri(self.classes[parent]["module"], - parent) + parent_uri = self.to_uri(self.classes[parent]["module"], parent) else: parent_mod = parent.split(".")[0] parent_uri = self.to_external_uri(parent_mod, parent) - mods[self.to_uri(cls["module"])]["namespaces"][ - cls_uri]["metadata"][ + mods[self.to_uri(cls["module"])]["namespaces"][cls_uri]["metadata"][ "superClasses" ].append(parent_uri) @@ -213,8 +206,7 @@ def add_superclasses(self, mods): def create_namespaces_map(self): namespaces_maps = [{}, {}] - for res, hmap in zip(namespaces_maps, [self.internal_mods, - self.external_mods]): + for res, hmap in zip(namespaces_maps, [self.internal_mods, self.external_mods]): for mod in hmap: for namespace in hmap[mod]["methods"]: res[namespace] = mod @@ -255,17 +247,13 @@ def get_graph(self): uris.append(self.namespace_map.get(uri, uri)) elif node in external: mod = external[node] - uris.append( - self.namespace_map.get(self.to_external_uri(mod, node)) - ) + uris.append(self.namespace_map.get(self.to_external_uri(mod, node))) if len(uris) == 2: if dst in external: - graph["externalCalls"].append([str(uris[0]), - str(uris[1]), {}]) + graph["externalCalls"].append([str(uris[0]), str(uris[1]), {}]) else: - graph["internalCalls"].append([str(uris[0]), - str(uris[1]), {}]) + graph["internalCalls"].append([str(uris[0]), str(uris[1]), {}]) return graph def generate(self): diff --git a/pycg/machinery/classes.py b/pycg/machinery/classes.py index dd2fd06..d9d46fe 100644 --- a/pycg/machinery/classes.py +++ b/pycg/machinery/classes.py @@ -53,7 +53,7 @@ def add_parent(self, parent): def fix_mro(self): new_mro = [] for idx, item in enumerate(self.mro): - if self.mro[idx + 1:].count(item) > 0: + if self.mro[idx + 1 :].count(item) > 0: continue new_mro.append(item) self.mro = new_mro diff --git a/pycg/machinery/definitions.py b/pycg/machinery/definitions.py index d2ec4f9..9e99420 100644 --- a/pycg/machinery/definitions.py +++ b/pycg/machinery/definitions.py @@ -44,8 +44,7 @@ def assign(self, ns, defi): # if it is a function def, we need to create a return pointer if defi.is_function_def(): return_ns = utils.join_ns(ns, utils.constants.RETURN_NAME) - self.defs[return_ns] = Definition(return_ns, - utils.constants.NAME_DEF) + self.defs[return_ns] = Definition(return_ns, utils.constants.NAME_DEF) self.defs[return_ns].get_name_pointer().add( utils.join_ns(defi.get_ns(), utils.constants.RETURN_NAME) ) @@ -164,29 +163,21 @@ def update_pointsto_args(pointsto_args, arg, name): pointsto_name_pointer = self.defs[name].get_name_pointer() # iterate the arguments of the definition # we're currently iterating - for arg_name, arg in ( - current_name_pointer.get_args().items() - ): + for arg_name, arg in current_name_pointer.get_args().items(): pos = current_name_pointer.get_pos_of_name(arg_name) if pos is not None: - pointsto_args = ( - pointsto_name_pointer.get_pos_arg(pos) - ) + pointsto_args = pointsto_name_pointer.get_pos_arg(pos) if not pointsto_args: - pointsto_name_pointer.add_pos_arg(pos, None, - arg) + pointsto_name_pointer.add_pos_arg(pos, None, arg) continue else: - pointsto_args = ( - pointsto_name_pointer.get_arg(arg_name) - ) + pointsto_args = pointsto_name_pointer.get_arg(arg_name) if not pointsto_args: pointsto_name_pointer.add_arg(arg_name, arg) continue - changed_something = changed_something or \ - update_pointsto_args( - pointsto_args, arg, current_def.get_ns() - ) + changed_something = changed_something or update_pointsto_args( + pointsto_args, arg, current_def.get_ns() + ) if not changed_something: break diff --git a/pycg/machinery/key_err.py b/pycg/machinery/key_err.py index 12fc042..3b1a860 100644 --- a/pycg/machinery/key_err.py +++ b/pycg/machinery/key_err.py @@ -24,8 +24,7 @@ def __init__(self): def add(self, filename, lineno, namespace, key): self.key_errs.append( - {"filename": filename, "lineno": lineno, "namespace": namespace, - "key": key} + {"filename": filename, "lineno": lineno, "namespace": namespace, "key": key} ) def get(self): diff --git a/pycg/machinery/scopes.py b/pycg/machinery/scopes.py index d8ba349..d4eb0f7 100644 --- a/pycg/machinery/scopes.py +++ b/pycg/machinery/scopes.py @@ -56,8 +56,7 @@ def process(namespace, parent, table): process(fullns, sc, t) process( - modulename, None, symtable.symtable(contents, filename, - compile_type="exec") + modulename, None, symtable.symtable(contents, filename, compile_type="exec") ) return {"functions": functions, "classes": classes} diff --git a/pycg/processing/base.py b/pycg/processing/base.py index d283492..6ccb1a1 100644 --- a/pycg/processing/base.py +++ b/pycg/processing/base.py @@ -90,12 +90,10 @@ def visit_For(self, node): self.visit(item) def visit_Dict(self, node): - counter = self.scope_manager.get_scope(self.current_ns). \ - inc_dict_counter() + counter = self.scope_manager.get_scope(self.current_ns).inc_dict_counter() dict_name = utils.get_dict_name(counter) - sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, - dict_name)) + sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, dict_name)) if not sc: return self.name_stack.append(dict_name) @@ -108,12 +106,10 @@ def visit_Dict(self, node): self.name_stack.pop() def visit_List(self, node): - counter = self.scope_manager.get_scope(self.current_ns). \ - inc_list_counter() + counter = self.scope_manager.get_scope(self.current_ns).inc_list_counter() list_name = utils.get_list_name(counter) - sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, - list_name)) + sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, list_name)) if not sc: return self.name_stack.append(list_name) @@ -189,8 +185,7 @@ def do_assign(decoded, target): self.visit(target) if isinstance(target, ast.Tuple): for pos, elt in enumerate(target.elts): - if not isinstance(decoded, - Definition) and pos < len(decoded): + if not isinstance(decoded, Definition) and pos < len(decoded): do_assign(decoded[pos], elt) else: targetns = self._get_target_ns(target) @@ -288,11 +283,7 @@ def decode_node(self, node): return [] def _is_literal(self, item): - return ( - isinstance(item, int) - or isinstance(item, str) - or isinstance(item, float) - ) + return isinstance(item, int) or isinstance(item, str) or isinstance(item, float) def _retrieve_base_names(self, node): if not isinstance(node, ast.Attribute): @@ -366,8 +357,7 @@ def _retrieve_attribute_names(self, node): continue ext_name = utils.join_ns(name, node.attr) if not self.def_manager.get(ext_name): - self.def_manager.create(ext_name, - utils.constants.EXT_DEF) + self.def_manager.create(ext_name, utils.constants.EXT_DEF) names.add(ext_name) return names @@ -392,8 +382,7 @@ def iterate_call_args(self, defi, node): else: for d in decoded: if isinstance(d, Definition): - defi.get_name_pointer().add_pos_arg(pos, None, - d.get_ns()) + defi.get_name_pointer().add_pos_arg(pos, None, d.get_ns()) else: defi.get_name_pointer().add_pos_lit_arg(pos, None, d) @@ -416,8 +405,7 @@ def iterate_call_args(self, defi, node): else: for d in decoded: if isinstance(d, Definition): - defi.get_name_pointer().add_arg(keyword.arg, - d.get_ns()) + defi.get_name_pointer().add_arg(keyword.arg, d.get_ns()) else: defi.get_name_pointer().add_lit_arg(keyword.arg, d) @@ -428,8 +416,7 @@ def retrieve_subscript_names(self, node): if not getattr(self, "closured", None): return set() - if getattr(node.slice, "value", None) and \ - self._is_literal(node.slice.value): + if getattr(node.slice, "value", None) and self._is_literal(node.slice.value): sl_names = [node.slice.value] else: sl_names = self.decode_node(node.slice) @@ -441,12 +428,10 @@ def retrieve_subscript_names(self, node): full_names = set() # get all names associated with this variable name for n in val_names: - if n and isinstance(n, Definition) and \ - self.closured.get(n.get_ns(), None): + if n and isinstance(n, Definition) and self.closured.get(n.get_ns(), None): decoded_vals |= self.closured.get(n.get_ns()) for s in sl_names: - if isinstance(s, Definition) and \ - self.closured.get(s.get_ns(), None): + if isinstance(s, Definition) and self.closured.get(s.get_ns(), None): # we care about the literals pointed by the name # not the namespaces, so retrieve the literals pointed for name in self.closured.get(s.get_ns()): @@ -553,8 +538,7 @@ def add_ext_mod_node(self, name): ext_modname = name.split(".")[0] ext_mod = self.module_manager.get(ext_modname) if not ext_mod: - ext_mod = self.module_manager.create(ext_modname, None, - external=True) + ext_mod = self.module_manager.create(ext_modname, None, external=True) ext_mod.add_method(ext_modname) ext_mod.add_method(name) diff --git a/pycg/processing/cgprocessor.py b/pycg/processing/cgprocessor.py index 855d6a3..ad8ed79 100644 --- a/pycg/processing/cgprocessor.py +++ b/pycg/processing/cgprocessor.py @@ -78,8 +78,7 @@ def visit_For(self, node): super().visit_For(node) def visit_Lambda(self, node): - counter = self.scope_manager.get_scope(self.current_ns).\ - inc_lambda_counter() + counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() lambda_name = utils.get_lambda_name(counter) lambda_fullns = utils.join_ns(self.current_ns, lambda_name) @@ -99,8 +98,7 @@ def visit_Raise(self, node): for name in names: pointer_def = self.def_manager.get(name) if pointer_def.get_type() == utils.constants.CLS_DEF: - init_ns = self.find_cls_fun_ns(name, - utils.constants.CLS_INIT) + init_ns = self.find_cls_fun_ns(name, utils.constants.CLS_INIT) for ns in init_ns: self.call_graph.add_edge(self.current_method, ns) if pointer_def.get_type() == utils.constants.EXT_DEF: @@ -144,18 +142,15 @@ def create_ext_edge(name, ext_modname): names = self.retrieve_call_names(node) if not names: - if isinstance(node.func, ast.Attribute) and \ - self.has_ext_parent(node.func): + if isinstance(node.func, ast.Attribute) and self.has_ext_parent(node.func): # TODO: This doesn't work for cases # where there is an assignment of an attribute # i.e. import os; lala = os.path; lala.dirname() for name in self.get_full_attr_names(node.func): ext_modname = name.split(".")[0] create_ext_edge(name, ext_modname) - elif getattr(node.func, "id", None) and \ - self.is_builtin(node.func.id): - name = utils.join_ns(utils.constants.BUILTIN_NAME, - node.func.id) + elif getattr(node.func, "id", None) and self.is_builtin(node.func.id): + name = utils.join_ns(utils.constants.BUILTIN_NAME, node.func.id) create_ext_edge(name, utils.constants.BUILTIN_NAME) return @@ -183,8 +178,7 @@ def create_ext_edge(name, ext_modname): # self.call_graph.add_edge(self.current_ns, dec_name) if pointer_def.get_type() == utils.constants.CLS_DEF: - init_ns = self.find_cls_fun_ns(pointer, - utils.constants.CLS_INIT) + init_ns = self.find_cls_fun_ns(pointer, utils.constants.CLS_INIT) for ns in init_ns: self.call_graph.add_edge(self.current_method, ns) diff --git a/pycg/processing/keyerrprocessor.py b/pycg/processing/keyerrprocessor.py index d8bcbcb..c469215 100644 --- a/pycg/processing/keyerrprocessor.py +++ b/pycg/processing/keyerrprocessor.py @@ -94,8 +94,7 @@ def analyze(self): self.analyze_submodules() def visit_Lambda(self, node): - counter = self.scope_manager.get_scope(self.current_ns).\ - inc_lambda_counter() + counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() lambda_name = utils.get_lambda_name(counter) utils.join_ns(self.current_ns, lambda_name) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index f04d6d0..f114e43 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -46,9 +46,7 @@ def __init__( self.closured = self.def_manager.transitive_closure() def visit_Lambda(self, node): - counter = self.scope_manager.\ - get_scope(self.current_ns).\ - inc_lambda_counter() + counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() lambda_name = utils.get_lambda_name(counter) super().visit_Lambda(node, lambda_name) @@ -110,8 +108,7 @@ def visit_For(self, node): ) ) if next_defi: - for name in self.closured.get(next_defi.get_ns(), - []): + for name in self.closured.get(next_defi.get_ns(), []): target_def.get_name_pointer().add(name) else: # otherwise, add a pointer to the name # (e.g. a yield) @@ -131,8 +128,7 @@ def visit_AsyncFunctionDef(self, node): def visit_FunctionDef(self, node): # here we iterate decorators if node.decorator_list: - fn_def = self.def_manager.get(utils.join_ns(self.current_ns, - node.name)) + fn_def = self.def_manager.get(utils.join_ns(self.current_ns, node.name)) reversed_decorators = list(reversed(node.decorator_list)) # add to the name pointer of the function definition @@ -157,8 +153,7 @@ def visit_FunctionDef(self, node): if not isinstance(d, Definition): continue for name in self.closured.get(d.get_ns(), []): - return_ns = utils.join_ns(name, - utils.constants.RETURN_NAME) + return_ns = utils.join_ns(name, utils.constants.RETURN_NAME) if self.closured.get(return_ns, None) is None: continue @@ -227,19 +222,16 @@ def visit_List(self, node): # create a list definition list_def = self.def_manager.get(list_full_ns) if not list_def: - list_def = self.def_manager.create(list_full_ns, - utils.constants.NAME_DEF) + list_def = self.def_manager.create(list_full_ns, utils.constants.NAME_DEF) current_scope.add_def(list_name, list_def) self.name_stack.append(list_name) for idx, elt in enumerate(node.elts): self.visit(elt) - key_full_ns = utils.join_ns(list_def.get_ns(), - utils.get_int_name(idx)) + key_full_ns = utils.join_ns(list_def.get_ns(), utils.get_int_name(idx)) key_def = self.def_manager.get(key_full_ns) if not key_def: - key_def = self.def_manager.create(key_full_ns, - utils.constants.NAME_DEF) + key_def = self.def_manager.create(key_full_ns, utils.constants.NAME_DEF) decoded_elt = self.decode_node(elt) for v in decoded_elt: @@ -260,14 +252,12 @@ def visit_Dict(self, node): dict_full_ns = utils.join_ns(self.current_ns, dict_name) # create a scope for the dict - dict_scope = self.scope_manager.create_scope(dict_full_ns, - current_scope) + dict_scope = self.scope_manager.create_scope(dict_full_ns, current_scope) # Create a dict definition dict_def = self.def_manager.get(dict_full_ns) if not dict_def: - dict_def = self.def_manager.create(dict_full_ns, - utils.constants.NAME_DEF) + dict_def = self.def_manager.create(dict_full_ns, utils.constants.NAME_DEF) # add it to the current scope current_scope.add_def(dict_name, dict_def) @@ -331,8 +321,7 @@ def update_parent_classes(self, defi): new_ns = utils.join_ns(parent_def.get_ns(), key) new_def = self.def_manager.get(new_ns) if not new_def: - new_def = self.def_manager.create(new_ns, - utils.constants.NAME_DEF) + new_def = self.def_manager.create(new_ns, utils.constants.NAME_DEF) new_def.get_name_pointer().add_set(names) new_def.get_name_pointer().add(child_def.get_ns()) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 08bfb7d..2776998 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -125,8 +125,7 @@ def iterate_mod_items(items, const): defi = self.def_manager.get(self.modname) if not defi: - defi = self.def_manager.create(self.modname, - utils.constants.MOD_DEF) + defi = self.def_manager.create(self.modname, utils.constants.MOD_DEF) super().visit_Module(node) @@ -163,9 +162,7 @@ def create_def(scope, name, imported_def): if tgt_name == "*": for name, defi in imported_scope.get_defs().items(): create_def(current_scope, name, defi) - current_scope.get_def(name).get_name_pointer().add( - defi.get_ns() - ) + current_scope.get_def(name).get_name_pointer().add(defi.get_ns()) else: # if it exists in the imported scope then copy it defi = imported_scope.get_def(imp_name) @@ -190,17 +187,13 @@ def add_external_def(name, target): tgt_ns = utils.join_ns(scope.get_ns(), target) tgt_defi = self.def_manager.get(tgt_ns) if not tgt_defi: - tgt_defi = self.def_manager.create( - tgt_ns, utils.constants.EXT_DEF - ) + tgt_defi = self.def_manager.create(tgt_ns, utils.constants.EXT_DEF) tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) for import_item in node.names: src_name = handle_src_name(import_item.name) - tgt_name = ( - import_item.asname if import_item.asname else import_item.name - ) + tgt_name = import_item.asname if import_item.asname else import_item.name imported_name = self.import_manager.handle_import(src_name, level) if not imported_name: @@ -284,8 +277,7 @@ def _handle_function_def(self, node, fn_name): arg_ns = utils.join_ns(fn_def.get_ns(), node.args.args[0].arg) arg_def = self.def_manager.get(arg_ns) if not arg_def: - arg_def = self.def_manager.create(arg_ns, - utils.constants.NAME_DEF) + arg_def = self.def_manager.create(arg_ns, utils.constants.NAME_DEF) arg_def.get_name_pointer().add(current_def.get_ns()) self.scope_manager.handle_assign( @@ -313,8 +305,7 @@ def _handle_function_def(self, node, fn_name): for arg_ns in defs_to_create: arg_def = self.def_manager.get(arg_ns) if not arg_def: - arg_def = self.def_manager.create(arg_ns, - utils.constants.NAME_DEF) + arg_def = self.def_manager.create(arg_ns, utils.constants.NAME_DEF) self.scope_manager.handle_assign( fn_def.get_ns(), arg_def.get_name(), arg_def @@ -347,8 +338,7 @@ def visit_For(self, node): if isinstance(node.target, ast.Name): target_ns = utils.join_ns(self.current_ns, node.target.id) if not self.def_manager.get(target_ns): - defi = self.def_manager.create(target_ns, - utils.constants.NAME_DEF) + defi = self.def_manager.create(target_ns, utils.constants.NAME_DEF) self.scope_manager.get_scope(self.current_ns).add_def( node.target.id, defi ) @@ -408,8 +398,7 @@ def visit_ClassDef(self, node): mod = self.module_manager.get(self.modname) if not mod: mod = self.module_manager.create(self.modname, self.filename) - mod.add_method(cls_def.get_ns(), node.lineno, - self._get_last_line(node)) + mod.add_method(cls_def.get_ns(), node.lineno, self._get_last_line(node)) # iterate bases to compute MRO for the class cls = self.class_manager.get(cls_def.get_ns()) diff --git a/pycg/tests/definitions_test.py b/pycg/tests/definitions_test.py index 94968bd..6c3975c 100644 --- a/pycg/tests/definitions_test.py +++ b/pycg/tests/definitions_test.py @@ -61,15 +61,13 @@ def test_assign(self): self.assertEqual(defi2.get_ns(), "defi2") # values should be merged self.assertEqual(defi2.get_type(), utils.constants.NAME_DEF) - self.assertEqual(defi2.get_name_pointer().get(), - set(["item1", "item2"])) + self.assertEqual(defi2.get_name_pointer().get(), set(["item1", "item2"])) self.assertEqual(defi2.get_name_pointer().get_arg(0), set(["arg"])) # for function defs a return def should be created too fndefi = dm.create("fndefi", utils.constants.FUN_DEF) dm.assign("fndefi2", fndefi) - return_def = dm.get("{}.{}".format("fndefi2", - utils.constants.RETURN_NAME)) + return_def = dm.get("{}.{}".format("fndefi2", utils.constants.RETURN_NAME)) self.assertIsNotNone(return_def) self.assertEqual( return_def.get_name_pointer().get(), diff --git a/pycg/tests/fasten_format_test.py b/pycg/tests/fasten_format_test.py index 1e3dcca..772c178 100644 --- a/pycg/tests/fasten_format_test.py +++ b/pycg/tests/fasten_format_test.py @@ -99,16 +99,14 @@ def test_uri(self): formatter.to_uri("mymod.mod1", "mymod.mod1.fn"), "/mymod.mod1/fn" ) self.assertEqual( - formatter.to_uri("mymod.mod1", "mymod.mod1.cls.fn"), - "/mymod.mod1/cls.fn" + formatter.to_uri("mymod.mod1", "mymod.mod1.cls.fn"), "/mymod.mod1/cls.fn" ) # test method starting with modname but without . inbetween with self.assertRaises(Exception): formatter.to_uri("mymod.mod1", "mymod.mod1cls.fn") # test method being in functions self.assertEqual( - formatter.to_uri("mod1.mod2", "mod1.mod2.myfunc"), - "/mod1.mod2/myfunc()" + formatter.to_uri("mod1.mod2", "mod1.mod2.myfunc"), "/mod1.mod2/myfunc()" ) # External uri check @@ -131,8 +129,7 @@ def _get_internal_mods(self): "mod1": { "filename": "mod1.py", "methods": { - "mod1.method": {"name": "mod1.method", "first": 2, - "last": 5}, + "mod1.method": {"name": "mod1.method", "first": 2, "last": 5}, "mod1.Cls.method": { "name": "mod1.Cls.method", "first": 6, @@ -156,8 +153,7 @@ def _get_internal_mods(self): "first": 6, "last": 9, }, - "mod.mod2.Cls": {"name": "mod.mod2.Cls", "first": 4, - "last": 9}, + "mod.mod2.Cls": {"name": "mod.mod2.Cls", "first": 4, "last": 9}, "mod.mod2.Cls.Nested": { "name": "mod.mod2.Cls.Nested", "first": 5, @@ -172,8 +168,7 @@ def _get_external_mods(self): "external": { "filename": None, "methods": { - "external": {"name": "external", "first": None, - "last": None}, + "external": {"name": "external", "first": None, "last": None}, "external.Cls": { "name": "external.Cls", "first": None, @@ -189,8 +184,7 @@ def _get_external_mods(self): "external2": { "filename": None, "methods": { - "external2": {"name": "external2", "first": None, - "last": None}, + "external2": {"name": "external2", "first": None, "last": None}, "external2.method": { "name": "external2.method", "first": None, @@ -203,8 +197,7 @@ def _get_external_mods(self): def _get_classes(self): return { "mod1.Cls": {"module": "mod1", "mro": ["mod1.Cls"]}, - "mod.mod2.Cls": {"module": "mod.mod2", "mro": ["mod.mod2.Cls", - "mod1.Cls"]}, + "mod.mod2.Cls": {"module": "mod.mod2", "mro": ["mod.mod2.Cls", "mod1.Cls"]}, "mod.mod2.Cls.Nested": { "module": "mod.mod2", "mro": ["mod.mod2.Cls.Nested", "external.Cls"], @@ -226,8 +219,7 @@ def test_internal_modules(self): # test that SourceFileName are correct for name, mod in internal_mods.items(): self.assertEqual( - mod["filename"], - internal_modules[formatter.to_uri(name)]["sourceFile"] + mod["filename"], internal_modules[formatter.to_uri(name)]["sourceFile"] ) # test that namespaces contains all methods @@ -241,15 +233,11 @@ def test_internal_modules(self): first = info["first"] last = info["last"] expected_namespaces.append( - dict(namespace=method_uri, metadata=dict(first=first, - last=last)) + dict(namespace=method_uri, metadata=dict(first=first, last=last)) ) # namespaces defined for module - result_namespaces = ( - internal_modules[name_uri]["namespaces"]. - values() - ) + result_namespaces = internal_modules[name_uri]["namespaces"].values() # unique identifiers defined for module result_ids = internal_modules[name_uri]["namespaces"].keys() @@ -268,10 +256,8 @@ def test_external_modules(self): external_modules = formatter.generate()["modules"]["external"] # test that external modules keys are identical with the ones generated - self.assertEqual(set(external_mods.keys()), - set(external_modules.keys())) - self.assertEqual(len(external_mods.keys()), - len(external_modules.keys())) + self.assertEqual(set(external_mods.keys()), set(external_modules.keys())) + self.assertEqual(len(external_mods.keys()), len(external_modules.keys())) # test that namespaces contains all the expected methods for name, mod in external_mods.items(): @@ -280,8 +266,7 @@ def test_external_modules(self): for method, info in mod["methods"].items(): if method != name: method_uri = formatter.to_external_uri(name, method) - expected_namespaces.append(dict(namespace=method_uri, - metadata={})) + expected_namespaces.append(dict(namespace=method_uri, metadata={})) # namespaces defined for external modules result_namespaces = external_modules[name]["namespaces"].values() @@ -314,8 +299,7 @@ def test_hiearchy(self): total_classes += 1 self.assertEqual(total_classes, len(classes.keys())) for cls_name, cls in classes.items(): - cls_name_uri = id_mapping[formatter.to_uri(cls["module"], - cls_name)] + cls_name_uri = id_mapping[formatter.to_uri(cls["module"], cls_name)] cls_mro = [] for item in cls["mro"]: # result mro should not contain the class name @@ -323,17 +307,13 @@ def test_hiearchy(self): continue if classes.get(item, None): # it is an internal module - cls_mro.append(formatter.to_uri(classes[item]["module"], - item)) + cls_mro.append(formatter.to_uri(classes[item]["module"], item)) else: - cls_mro.append( - formatter.to_external_uri(item.split(".")[0], - item)) + cls_mro.append(formatter.to_external_uri(item.split(".")[0], item)) self.assertEqual( cls_mro, - modules[formatter.to_uri(cls["module"])]["namespaces"] - [cls_name_uri][ + modules[formatter.to_uri(cls["module"])]["namespaces"][cls_name_uri][ "metadata" ]["superClasses"], ) diff --git a/pycg/tests/imports_test.py b/pycg/tests/imports_test.py index 0d495ab..9209841 100644 --- a/pycg/tests/imports_test.py +++ b/pycg/tests/imports_test.py @@ -106,14 +106,12 @@ def test_hooks(self): custom_loader = "custom_loader" with mock.patch( - "importlib.machinery.FileFinder.path_hook", - return_value=custom_loader + "importlib.machinery.FileFinder.path_hook", return_value=custom_loader ): im.install_hooks() self.assertEqual(sys.path_hooks[0], custom_loader) - self.assertEqual(sys.path[0], - os.path.abspath(os.path.dirname(input_file))) + self.assertEqual(sys.path[0], os.path.abspath(os.path.dirname(input_file))) im.remove_hooks() self.assertEqual(old_sys_path, sys.path) @@ -151,12 +149,9 @@ def test_handle_import_level(self): with self.assertRaises(ImportError): im._handle_import_level("something", 4) - self.assertEqual(im._handle_import_level("smth", 2), - ("..smth", "mod1")) - self.assertEqual(im._handle_import_level("smth", 1), - (".smth", "mod1.mod2")) - self.assertEqual(im._handle_import_level("smth", 0), - ("smth", "")) + self.assertEqual(im._handle_import_level("smth", 2), ("..smth", "mod1")) + self.assertEqual(im._handle_import_level("smth", 1), (".smth", "mod1.mod2")) + self.assertEqual(im._handle_import_level("smth", 0), ("smth", "")) def test_handle_import(self): # test builtin modules diff --git a/pycg/tests/pointers_test.py b/pycg/tests/pointers_test.py index c0e143b..4eef454 100644 --- a/pycg/tests/pointers_test.py +++ b/pycg/tests/pointers_test.py @@ -108,6 +108,5 @@ def test_name_pointer_merge(self): pointer1.merge(pointer2) self.assertEqual(pointer1.get(), set(["smth1", "smth6"])) - self.assertEqual(pointer1.get_arg(0), - set(["smth2", "smth3", "smth7", "smth8"])) + self.assertEqual(pointer1.get_arg(0), set(["smth2", "smth3", "smth7", "smth8"])) self.assertEqual(pointer1.get_arg(1), set(["smth4", "smth9"])) diff --git a/pycg/tests/scopes_test.py b/pycg/tests/scopes_test.py index 4ea89a6..c14d67b 100644 --- a/pycg/tests/scopes_test.py +++ b/pycg/tests/scopes_test.py @@ -77,42 +77,33 @@ def get_lineno(self): self.assertEqual(sm.get_scope("root").parent, None) self.assertEqual(sm.get_scope("root.chld1").get_ns(), "root.chld1") - self.assertEqual(sm.get_scope("root.chld1").parent, - sm.get_scope("root")) + self.assertEqual(sm.get_scope("root.chld1").parent, sm.get_scope("root")) self.assertEqual(sm.get_scope("root.chld2").get_ns(), "root.chld2") - self.assertEqual(sm.get_scope("root.chld2").parent, - sm.get_scope("root")) + self.assertEqual(sm.get_scope("root.chld2").parent, sm.get_scope("root")) self.assertEqual(sm.get_scope("root.chld3").get_ns(), "root.chld3") - self.assertEqual(sm.get_scope("root.chld3").parent, - sm.get_scope("root")) + self.assertEqual(sm.get_scope("root.chld3").parent, sm.get_scope("root")) self.assertEqual( - sm.get_scope("root.chld1.grndchld1").get_ns(), - "root.chld1.grndchld1" + sm.get_scope("root.chld1.grndchld1").get_ns(), "root.chld1.grndchld1" ) self.assertEqual( - sm.get_scope("root.chld1.grndchld1").parent, - sm.get_scope("root.chld1") + sm.get_scope("root.chld1.grndchld1").parent, sm.get_scope("root.chld1") ) self.assertEqual( - sm.get_scope("root.chld1.grndchld2").get_ns(), - "root.chld1.grndchld2" + sm.get_scope("root.chld1.grndchld2").get_ns(), "root.chld1.grndchld2" ) self.assertEqual( - sm.get_scope("root.chld1.grndchld2").parent, - sm.get_scope("root.chld1") + sm.get_scope("root.chld1.grndchld2").parent, sm.get_scope("root.chld1") ) self.assertEqual( - sm.get_scope("root.chld2.grndchld3").get_ns(), - "root.chld2.grndchld3" + sm.get_scope("root.chld2.grndchld3").get_ns(), "root.chld2.grndchld3" ) self.assertEqual( - sm.get_scope("root.chld2.grndchld3").parent, - sm.get_scope("root.chld2") + sm.get_scope("root.chld2.grndchld3").parent, sm.get_scope("root.chld2") ) def test_handle_assign(self): @@ -131,8 +122,7 @@ def test_get_def(self): sm.scopes[root] = ScopeItem(root, None) # root scope sm.scopes[chld1] = ScopeItem(chld1, sm.scopes[root]) # 1st child scope sm.scopes[chld2] = ScopeItem(chld2, sm.scopes[root]) # 2nd child scope - sm.scopes[grndchld] = ScopeItem(grndchld, - sm.scopes[chld1]) # grandchild + sm.scopes[grndchld] = ScopeItem(grndchld, sm.scopes[chld1]) # grandchild grndchld_def = ("var", "grndchild_def") # name, value chld1_def1 = ("var", "chld1_def") @@ -148,20 +138,15 @@ def test_get_def(self): # should be able to find a variable defined in its scope # also it should get the value of the nearest scope, meaning it's own - self.assertEqual(sm.get_def("ns.chld1.chld1", - grndchld_def[0]), grndchld_def[1]) + self.assertEqual(sm.get_def("ns.chld1.chld1", grndchld_def[0]), grndchld_def[1]) # if it doesn't exist get the parent's - self.assertEqual(sm.get_def("ns.chld1.chld1", - chld1_def2[0]), chld1_def2[1]) + self.assertEqual(sm.get_def("ns.chld1.chld1", chld1_def2[0]), chld1_def2[1]) # it shouldn't be able to reach a def defined in chld2 - self.assertEqual(sm.get_def("ns.chld1.chld1", - chld2_def[0]), None) + self.assertEqual(sm.get_def("ns.chld1.chld1", chld2_def[0]), None) # root doesn't have access to variables defined in lower scopes - self.assertEqual(sm.get_def("ns", - chld1_def2[0]), None) + self.assertEqual(sm.get_def("ns", chld1_def2[0]), None) # but childs have access to root - self.assertEqual(sm.get_def("ns.chld2", - root_def[0]), root_def[1]) + self.assertEqual(sm.get_def("ns.chld2", root_def[0]), root_def[1]) def test_get_scope(self): sm = ScopeManager() diff --git a/setup.py b/setup.py index 4c04ec8..e6e7b18 100644 --- a/setup.py +++ b/setup.py @@ -23,36 +23,39 @@ from setuptools import setup, find_packages from subprocess import call + def get_long_desc(): with open("README.md", "r") as readme: desc = readme.read() return desc + def setup_package(): setup( - name='pycg', - version='0.0.6', - description='Practical Python Call Graphs', + name="pycg", + version="0.0.6", + description="Practical Python Call Graphs", long_description=get_long_desc(), long_description_content_type="text/markdown", - url='https://github.com/vitsalis/pycg', - license='Apache Software License', + url="https://github.com/vitsalis/pycg", + license="Apache Software License", packages=find_packages(), install_requires=[], - python_requires='>=3.4', - entry_points = { - 'console_scripts': [ - 'pycg=pycg.__main__:main', + python_requires=">=3.4", + entry_points={ + "console_scripts": [ + "pycg=pycg.__main__:main", ], }, classifiers=[ - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3' + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", ], - author = 'Vitalis Salis', - author_email = 'vitsalis@gmail.com' + author="Vitalis Salis", + author_email="vitsalis@gmail.com", ) -if __name__ == '__main__': + +if __name__ == "__main__": setup_package() From 17286bc4d23f8ebaa8e35b0a7795e8fa589e3c3a Mon Sep 17 00:00:00 2001 From: Giorgos Drosos Date: Tue, 16 May 2023 16:36:58 +0300 Subject: [PATCH 7/7] Ignore 2 flake8 rules in conflict with black --- .github/workflows/linters.yml | 2 +- pycg/__main__.py | 8 +++----- pycg/formats/fasten.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index e8290cd..d83c30c 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -31,7 +31,7 @@ jobs: uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2 with: linters: flake8 - run: flake8 --max-line-length 89 pycg + run: flake8 --max-line-length 89 --ignore=E203,W503 pycg isort: runs-on: ubuntu-latest diff --git a/pycg/__main__.py b/pycg/__main__.py index a109308..fb254c3 100644 --- a/pycg/__main__.py +++ b/pycg/__main__.py @@ -31,7 +31,7 @@ def main(): type=int, help=( "Maximum number of iterations through source code. " - + "If not specified a fix-point iteration will be performed." + "If not specified a fix-point iteration will be performed." ), default=-1, ) @@ -40,11 +40,9 @@ def main(): type=str, choices=[CALL_GRAPH_OP, KEY_ERR_OP], help=( - "Operation to perform. " - + "Choose " + "Operation to perform. Choose " + CALL_GRAPH_OP - + " for call graph generation (default)" - + " or " + + " for call graph generation (default) or " + KEY_ERR_OP + " for key error detection on dictionaries." ), diff --git a/pycg/formats/fasten.py b/pycg/formats/fasten.py index 3d56f56..1c84569 100644 --- a/pycg/formats/fasten.py +++ b/pycg/formats/fasten.py @@ -57,7 +57,7 @@ def to_uri(self, modname, name=""): if not name.startswith(modname + "."): raise Exception("name should start with modname", name, modname) - cleared = name[len(modname) + 1 :] + cleared = name[(len(modname) + 1) :] suffix = "" if name in self.functions: