From 1f25ed44ea700c814fc306f423c605d5a26f0454 Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Sat, 28 Sep 2019 13:21:36 -0400 Subject: [PATCH 01/11] compeltes the Stack implementation to yield ending line for each class. --- Lib/pyclbr.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 99a17343fb61fd..9a504e621cf5e2 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -51,6 +51,7 @@ class _Object: "Information about Python class or function." + def __init__(self, module, name, file, lineno, parent): self.module = module self.name = name @@ -65,12 +66,14 @@ def _addchild(self, name, obj): class Function(_Object): "Information about a Python function, including methods." + def __init__(self, module, name, file, lineno, parent=None): _Object.__init__(self, module, name, file, lineno, parent) class Class(_Object): "Information about a Python class." + def __init__(self, module, name, super, file, lineno, parent=None): _Object.__init__(self, module, name, file, lineno, parent) self.super = [] if super is None else super @@ -80,6 +83,20 @@ def _addmethod(self, name, lineno): self.methods[name] = lineno +class Stack(list): + + def __init__(self): + import inspect + super().__init__() + + def __delitem__(self, key): + import inspect + frames = inspect.stack() + setattr(self[key][0], 'endline', + frames[1].frame.f_locals['start'][0] - 1) + super().__delitem__(key) + + def _nest_function(ob, func_name, lineno): "Return a Function after nesting within ob." newfunc = Function(ob.module, func_name, ob.file, lineno, ob) @@ -88,12 +105,14 @@ def _nest_function(ob, func_name, lineno): ob._addmethod(func_name, lineno) return newfunc + def _nest_class(ob, class_name, lineno, super=None): "Return a Class after nesting within ob." newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) ob._addchild(class_name, newclass) return newclass + def readmodule(module, path=None): """Return Class objects for the top-level classes in module. @@ -106,6 +125,7 @@ def readmodule(module, path=None): res[key] = value return res + def readmodule_ex(module, path=None): """Return a dictionary with all functions and classes in module. @@ -115,6 +135,7 @@ def readmodule_ex(module, path=None): """ return _readmodule(module, path or []) + def _readmodule(module, path, inpackage=None): """Do the hard work for readmodule[_ex]. @@ -161,7 +182,8 @@ def _readmodule(module, path, inpackage=None): search_path = path + sys.path spec = importlib.util._find_spec_from_path(fullmodule, search_path) if spec is None: - raise ModuleNotFoundError(f"no module named {fullmodule!r}", name=fullmodule) + raise ModuleNotFoundError( + f"no module named {fullmodule!r}", name=fullmodule) _modules[fullmodule] = tree # Is module a package? if spec.submodule_search_locations is not None: @@ -193,8 +215,8 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): """ f = io.StringIO(source) - stack = [] # Initialize stack of (class, indent) pairs. - + # stack = [] # Initialize stack of (class, indent) pairs. + stack = Stack() # changing the source code so as to get the ending line g = tokenize.generate_tokens(f.readline) try: for tokentype, token, start, _end, _line in g: @@ -227,14 +249,14 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: - continue # Skip class with syntax error. + continue # Skip class with syntax error. # Parse what follows the class name. tokentype, token, start = next(g)[0:3] inherit = None if token == '(': - names = [] # Initialize list of superclasses. + names = [] # Initialize list of superclasses. level = 1 - super = [] # Tokens making up current superclass. + super = [] # Tokens making up current superclass. while True: tokentype, token, start = next(g)[0:3] if token in (')', ',') and level == 1: @@ -271,7 +293,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): if stack: cur_obj = stack[-1][0] cur_class = _nest_class( - cur_obj, class_name, lineno, inherit) + cur_obj, class_name, lineno, inherit) else: cur_class = Class(fullmodule, class_name, inherit, fname, lineno) @@ -377,7 +399,7 @@ def _main(): else: path = [] tree = readmodule_ex(mod, path) - lineno_key = lambda a: getattr(a, 'lineno', 0) + def lineno_key(a): return getattr(a, 'lineno', 0) objs = sorted(tree.values(), key=lineno_key, reverse=True) indent_level = 2 while objs: @@ -400,5 +422,6 @@ def _main(): elif isinstance(obj, Function): print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno)) + if __name__ == "__main__": _main() From 7d2354ac2f1a9b83f97d5602b0b910f20d823592 Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Sun, 6 Oct 2019 15:42:30 -0400 Subject: [PATCH 02/11] reverts whitespace and unwanted changes including functions, comments and formatting --- Lib/pyclbr.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 9a504e621cf5e2..38e42f50f36531 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -43,6 +43,7 @@ import importlib.util import tokenize from token import NAME, DEDENT, OP +import inspect __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] @@ -51,7 +52,6 @@ class _Object: "Information about Python class or function." - def __init__(self, module, name, file, lineno, parent): self.module = module self.name = name @@ -84,13 +84,8 @@ def _addmethod(self, name, lineno): class Stack(list): - - def __init__(self): - import inspect - super().__init__() - + def __delitem__(self, key): - import inspect frames = inspect.stack() setattr(self[key][0], 'endline', frames[1].frame.f_locals['start'][0] - 1) @@ -182,8 +177,7 @@ def _readmodule(module, path, inpackage=None): search_path = path + sys.path spec = importlib.util._find_spec_from_path(fullmodule, search_path) if spec is None: - raise ModuleNotFoundError( - f"no module named {fullmodule!r}", name=fullmodule) + raise ModuleNotFoundError(f"no module named {fullmodule!r}", name=fullmodule) _modules[fullmodule] = tree # Is module a package? if spec.submodule_search_locations is not None: @@ -215,8 +209,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): """ f = io.StringIO(source) - # stack = [] # Initialize stack of (class, indent) pairs. - stack = Stack() # changing the source code so as to get the ending line + stack = Stack() # Initialize stack of (class, indent) pairs. g = tokenize.generate_tokens(f.readline) try: for tokentype, token, start, _end, _line in g: @@ -249,14 +242,14 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: - continue # Skip class with syntax error. + continue # Skip class with syntax error. # Parse what follows the class name. tokentype, token, start = next(g)[0:3] inherit = None if token == '(': - names = [] # Initialize list of superclasses. + names = [] # Initialize list of superclasses. level = 1 - super = [] # Tokens making up current superclass. + super = [] # Tokens making up current superclass. while True: tokentype, token, start = next(g)[0:3] if token in (')', ',') and level == 1: @@ -293,7 +286,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): if stack: cur_obj = stack[-1][0] cur_class = _nest_class( - cur_obj, class_name, lineno, inherit) + cur_obj, class_name, lineno, inherit) else: cur_class = Class(fullmodule, class_name, inherit, fname, lineno) @@ -399,7 +392,7 @@ def _main(): else: path = [] tree = readmodule_ex(mod, path) - def lineno_key(a): return getattr(a, 'lineno', 0) + lineno_key = lambda a: getattr(a, 'lineno', 0) objs = sorted(tree.values(), key=lineno_key, reverse=True) indent_level = 2 while objs: @@ -422,6 +415,5 @@ def lineno_key(a): return getattr(a, 'lineno', 0) elif isinstance(obj, Function): print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno)) - if __name__ == "__main__": _main() From c8a003d2264f7ddd1c4117c1558c07a09ccd5e25 Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Sun, 6 Oct 2019 15:46:32 -0400 Subject: [PATCH 03/11] removes leftover whitespace in b/w functions: unwanted --- Lib/pyclbr.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 38e42f50f36531..06600a9024d8d3 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -66,14 +66,12 @@ def _addchild(self, name, obj): class Function(_Object): "Information about a Python function, including methods." - def __init__(self, module, name, file, lineno, parent=None): _Object.__init__(self, module, name, file, lineno, parent) class Class(_Object): "Information about a Python class." - def __init__(self, module, name, super, file, lineno, parent=None): _Object.__init__(self, module, name, file, lineno, parent) self.super = [] if super is None else super @@ -100,14 +98,12 @@ def _nest_function(ob, func_name, lineno): ob._addmethod(func_name, lineno) return newfunc - def _nest_class(ob, class_name, lineno, super=None): "Return a Class after nesting within ob." newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) ob._addchild(class_name, newclass) return newclass - def readmodule(module, path=None): """Return Class objects for the top-level classes in module. @@ -120,7 +116,6 @@ def readmodule(module, path=None): res[key] = value return res - def readmodule_ex(module, path=None): """Return a dictionary with all functions and classes in module. @@ -130,7 +125,6 @@ def readmodule_ex(module, path=None): """ return _readmodule(module, path or []) - def _readmodule(module, path, inpackage=None): """Do the hard work for readmodule[_ex]. @@ -210,6 +204,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): f = io.StringIO(source) stack = Stack() # Initialize stack of (class, indent) pairs. + g = tokenize.generate_tokens(f.readline) try: for tokentype, token, start, _end, _line in g: From 7e6f079b7af7c5ada606797785934271c0cf9e20 Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Mon, 3 Feb 2020 20:36:51 -0500 Subject: [PATCH 04/11] Initialize stack of (class, indent) pairs. --- Lib/pyclbr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 06600a9024d8d3..e7f109082fb8b3 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -203,7 +203,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): """ f = io.StringIO(source) - stack = Stack() # Initialize stack of (class, indent) pairs. + stack = Stack() g = tokenize.generate_tokens(f.readline) try: From b7c1ce7b7101196d33318350fe4b8553257e6a33 Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Mon, 3 Feb 2020 20:44:36 -0500 Subject: [PATCH 05/11] removes Stack() custom class and adds direct attribute (end_lineno) instead to the stack --- Lib/pyclbr.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index e7f109082fb8b3..1910f2aa1b816e 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -81,15 +81,6 @@ def _addmethod(self, name, lineno): self.methods[name] = lineno -class Stack(list): - - def __delitem__(self, key): - frames = inspect.stack() - setattr(self[key][0], 'endline', - frames[1].frame.f_locals['start'][0] - 1) - super().__delitem__(key) - - def _nest_function(ob, func_name, lineno): "Return a Function after nesting within ob." newfunc = Function(ob.module, func_name, ob.file, lineno, ob) @@ -203,7 +194,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): """ f = io.StringIO(source) - stack = Stack() + stack = [] g = tokenize.generate_tokens(f.readline) try: @@ -212,11 +203,13 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: + stack[-1].end_lineno = start - 1 del stack[-1] elif token == 'def': lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: + stack[-1].end_lineno = start - 1 del stack[-1] tokentype, func_name, start = next(g)[0:3] if tokentype != NAME: @@ -234,6 +227,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: + stack[-1].end_lineno = start - 1 del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: From 3466f638f3d43e8dbe59d44ba977edb5d772cd39 Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Fri, 7 Feb 2020 21:01:22 -0500 Subject: [PATCH 06/11] corrects the attribute name to end_lineno and sets the same without using subclass --- Lib/pyclbr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 1910f2aa1b816e..9ef3a5f0336f9c 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -203,13 +203,13 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: - stack[-1].end_lineno = start - 1 + setattr(stack[-1][0], 'end_lineno', start[0] - 1) del stack[-1] elif token == 'def': lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: - stack[-1].end_lineno = start - 1 + setattr(stack[-1][0], 'end_lineno', start[0] - 1) del stack[-1] tokentype, func_name, start = next(g)[0:3] if tokentype != NAME: @@ -227,7 +227,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: - stack[-1].end_lineno = start - 1 + setattr(stack[-1][0], 'end_lineno', start[0] - 1) del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: From f2a58c68bfed1b0b5bac743f4bcb1b3069fdb08a Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Fri, 7 Feb 2020 21:03:13 -0500 Subject: [PATCH 07/11] removes unwanted module: inspect --- Lib/pyclbr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 9ef3a5f0336f9c..8f885d6b31c5ca 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -43,7 +43,6 @@ import importlib.util import tokenize from token import NAME, DEDENT, OP -import inspect __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] @@ -194,7 +193,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): """ f = io.StringIO(source) - stack = [] + stack = [] # Initialize stack of (class, indent) pairs. g = tokenize.generate_tokens(f.readline) try: From 94edb5bae1eb321049494276b1cdfeb6364d5c2d Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Sun, 16 Feb 2020 11:10:20 -0500 Subject: [PATCH 08/11] corrects the way of setting attribute --- Lib/pyclbr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 8f885d6b31c5ca..f49cc1a8f79cb0 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -202,13 +202,13 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: - setattr(stack[-1][0], 'end_lineno', start[0] - 1) + stack[-1][0].end_lineno = start[0] - 1 del stack[-1] elif token == 'def': lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: - setattr(stack[-1][0], 'end_lineno', start[0] - 1) + stack[-1][0].end_lineno = start[0] - 1 del stack[-1] tokentype, func_name, start = next(g)[0:3] if tokentype != NAME: @@ -226,7 +226,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: - setattr(stack[-1][0], 'end_lineno', start[0] - 1) + stack[-1][0].end_lineno = start[0] - 1 del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: From 454d30540799fede401779c1c32039f0c3283b4d Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2020 03:03:22 +0000 Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2020-03-16-03-03-21.bpo-38307.2cmw2i.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-03-16-03-03-21.bpo-38307.2cmw2i.rst diff --git a/Misc/NEWS.d/next/Library/2020-03-16-03-03-21.bpo-38307.2cmw2i.rst b/Misc/NEWS.d/next/Library/2020-03-16-03-03-21.bpo-38307.2cmw2i.rst new file mode 100644 index 00000000000000..06d476a7326cd2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-03-16-03-03-21.bpo-38307.2cmw2i.rst @@ -0,0 +1,4 @@ +Adds end line no in class' use: generating dependency. +: `end_lineno` is added to denote the scope of a class +along with `lineno` which was already present in the codebase. +Patch by Aviral Srivastava. \ No newline at end of file From dc0166f6abd4e0a5254e333bd89a08df8f6e4e54 Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Mon, 23 Mar 2020 20:49:43 -0400 Subject: [PATCH 10/11] wip: added end_lineno in tests --- Lib/test/test_pyclbr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 4385271cd0f2ba..653ae420a6a9a8 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -206,8 +206,8 @@ def compare(parent1, children1, parent2, children2): self.assertIs(ob.parent, parent2) for key in children1.keys(): o1, o2 = children1[key], children2[key] - t1 = type(o1), o1.name, o1.file, o1.module, o1.lineno - t2 = type(o2), o2.name, o2.file, o2.module, o2.lineno + t1 = type(o1), o1.name, o1.file, o1.module, o1.lineno, o1.end_lineno + t2 = type(o2), o2.name, o2.file, o2.module, o2.lineno, o2.end_lineno self.assertEqual(t1, t2) if type(o1) is mb.Class: self.assertEqual(o1.methods, o2.methods) From ea2090b89f1705646e78b999e34c0ac7b6e410dd Mon Sep 17 00:00:00 2001 From: Aviral Srivastava Date: Thu, 13 Aug 2020 19:44:18 -0400 Subject: [PATCH 11/11] wip --- Lib/pyclbr.py | 1 + Lib/test/test_pyclbr.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index f49cc1a8f79cb0..247d857651a471 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -21,6 +21,7 @@ name -- name of the object; file -- file in which the object is defined; lineno -- line in the file where the object's definition starts; + end_lineno -- line in the file where the object's definition ends; parent -- parent of this object, if any; children -- nested objects contained in this object. The 'children' attribute is a dictionary mapping names to objects. diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 653ae420a6a9a8..53a004dc1aef0a 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -199,6 +199,8 @@ def compare(parent1, children1, parent2, children2): linkage. We separate comparing string and number attributes from comparing the children of input children. """ + print("Children 1 is: {}".format(children1)) + print("Children 2 is: {}".format(children2)) self.assertEqual(children1.keys(), children2.keys()) for ob in children1.values(): self.assertIs(ob.parent, parent1)