From 366dc1a2f7992e5fcee68c3dbd99e4f73a2c95ce Mon Sep 17 00:00:00 2001 From: Robert Kalinowski Date: Thu, 4 Jul 2019 15:11:33 +0200 Subject: [PATCH 1/7] FIX: Handle ro-value-descrptiors (#76) * FIX: Handle read-only value descriptors. --- pdoc/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdoc/__init__.py b/pdoc/__init__.py index be47cf98..b1ada4a8 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -819,7 +819,7 @@ def __init__(self, name, module, obj, *, docstring=None): for name, obj in public_objs: if name in self.doc and self.doc[name].docstring: continue - if inspect.isroutine(obj): + if inspect.isroutine(obj) and callable(obj): self.doc[name] = Function( name, self.module, obj, cls=self, method=not self._method_type(self.obj, name)) From 5d2c69dab8926d97fab33eebe008d7a9abb7045d Mon Sep 17 00:00:00 2001 From: Robert Kalinowski Date: Fri, 5 Jul 2019 11:09:56 +0200 Subject: [PATCH 2/7] 'FIX: ro-value-descrptiors as instance var(#76)' --- pdoc/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pdoc/__init__.py b/pdoc/__init__.py index b1ada4a8..dac36eb7 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -818,6 +818,9 @@ def __init__(self, name, module, obj, *, docstring=None): # Convert the public Python objects to documentation objects. for name, obj in public_objs: if name in self.doc and self.doc[name].docstring: + if inspect.isroutine(obj) and not callable(obj): + assert isinstance(self.doc[name], Variable) + self.doc[name].instance_var = True continue if inspect.isroutine(obj) and callable(obj): self.doc[name] = Function( @@ -825,7 +828,8 @@ def __init__(self, name, module, obj, *, docstring=None): method=not self._method_type(self.obj, name)) elif (inspect.isdatadescriptor(obj) or inspect.isgetsetdescriptor(obj) or - inspect.ismemberdescriptor(obj)): + inspect.ismemberdescriptor(obj) or + inspect.isroutine(obj)): self.doc[name] = Variable( name, self.module, inspect.getdoc(obj), obj=getattr(obj, 'fget', obj), From 101b19b5bdfab4197cefa4d401acfa785e668fd5 Mon Sep 17 00:00:00 2001 From: Robert Kalinowski Date: Fri, 5 Jul 2019 11:12:09 +0200 Subject: [PATCH 3/7] FIX: unittests for ro-value-descrptiors (#76) --- pdoc/test/__init__.py | 17 +++++++++++++++++ pdoc/test/example_pkg/__init__.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py index 147d0484..edbedc72 100644 --- a/pdoc/test/__init__.py +++ b/pdoc/test/__init__.py @@ -414,6 +414,23 @@ def test_instance_var(self): var = mod.doc['B'].doc['instance_var'] self.assertTrue(var.instance_var) + def test_readonly_var_descriptors(self): + pdoc.reset() + mod = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE)) + var = mod.doc['B'].doc['squere'] + self.assertIsInstance(var, pdoc.Variable) + self.assertTrue(var.instance_var) + self.assertEqual(var.docstring, """Squere of variable `var`""") + + def test_readonly_var_descriptors_no_docstring(self): + pdoc.reset() + mod = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE)) + var = mod.doc['B'].doc['squere_no_doc'] + self.assertIsInstance(var, pdoc.Variable) + self.assertTrue(var.instance_var) + self.assertEqual(var.docstring, + """Read-only value descriptor, returns squere of variable `var`""") + def test_builtin_methoddescriptors(self): import parser with self.assertWarns(UserWarning): diff --git a/pdoc/test/example_pkg/__init__.py b/pdoc/test/example_pkg/__init__.py index 55fa0ef1..20edb781 100644 --- a/pdoc/test/example_pkg/__init__.py +++ b/pdoc/test/example_pkg/__init__.py @@ -20,6 +20,15 @@ def foo(env=os.environ): pass +class ReadOnlyValueDescrpitor: + """Read-only value descriptor, returns squere of variable `var`""" + + def __get__(self, instance, instance_type=None): + if instance is not None: + return instance.var ** 2 + return self + + class A: """`A` is base class for `example_pkg.B`.""" # Test refname link def overridden(self): @@ -45,6 +54,11 @@ class B(A, int): var = 3 """B.var docstring""" + squere = ReadOnlyValueDescrpitor() + """Squere of variable `var`""" + + squere_no_doc = ReadOnlyValueDescrpitor() # no doc-string + def __init__(self, x, y, z, w): """__init__ docstring""" self.instance_var = None From 9aada78d30be140dc18b0d298756f09e720334c1 Mon Sep 17 00:00:00 2001 From: Robert Kalinowski Date: Fri, 5 Jul 2019 11:27:49 +0200 Subject: [PATCH 4/7] FIX: separate section for ro-value-descrptiors (#76) --- pdoc/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pdoc/__init__.py b/pdoc/__init__.py index dac36eb7..c499f0ee 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -826,10 +826,14 @@ def __init__(self, name, module, obj, *, docstring=None): self.doc[name] = Function( name, self.module, obj, cls=self, method=not self._method_type(self.obj, name)) + elif inspect.isroutine(obj): + self.doc[name] = Variable( + name, self.module, inspect.getdoc(obj), + obj=getattr(obj, '__get__', obj), + cls=self, instance_var=True) elif (inspect.isdatadescriptor(obj) or inspect.isgetsetdescriptor(obj) or - inspect.ismemberdescriptor(obj) or - inspect.isroutine(obj)): + inspect.ismemberdescriptor(obj)): self.doc[name] = Variable( name, self.module, inspect.getdoc(obj), obj=getattr(obj, 'fget', obj), From 48039a575521fee90ba61005c89401534b99a3ec Mon Sep 17 00:00:00 2001 From: Robert Kalinowski Date: Mon, 8 Jul 2019 12:59:20 +0200 Subject: [PATCH 5/7] FIX: unittest for ro-value-descrptiors (#76) --- pdoc/test/__init__.py | 11 ++++------- pdoc/test/example_pkg/__init__.py | 10 +++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py index edbedc72..31f2eb38 100644 --- a/pdoc/test/__init__.py +++ b/pdoc/test/__init__.py @@ -417,19 +417,16 @@ def test_instance_var(self): def test_readonly_var_descriptors(self): pdoc.reset() mod = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE)) - var = mod.doc['B'].doc['squere'] + var = mod.doc['B'].doc['square'] self.assertIsInstance(var, pdoc.Variable) self.assertTrue(var.instance_var) - self.assertEqual(var.docstring, """Squere of variable `var`""") - - def test_readonly_var_descriptors_no_docstring(self): - pdoc.reset() - mod = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE)) + self.assertEqual(var.docstring, """Square of variable `var`""") var = mod.doc['B'].doc['squere_no_doc'] self.assertIsInstance(var, pdoc.Variable) self.assertTrue(var.instance_var) self.assertEqual(var.docstring, - """Read-only value descriptor, returns squere of variable `var`""") + """Read-only value descriptor, returns square of variable `var`""") + self.assertTrue(var.source) def test_builtin_methoddescriptors(self): import parser diff --git a/pdoc/test/example_pkg/__init__.py b/pdoc/test/example_pkg/__init__.py index 20edb781..bf14ebe2 100644 --- a/pdoc/test/example_pkg/__init__.py +++ b/pdoc/test/example_pkg/__init__.py @@ -20,8 +20,8 @@ def foo(env=os.environ): pass -class ReadOnlyValueDescrpitor: - """Read-only value descriptor, returns squere of variable `var`""" +class ReadOnlyValueDescriptor: + """Read-only value descriptor, returns square of variable `var`""" def __get__(self, instance, instance_type=None): if instance is not None: @@ -54,10 +54,10 @@ class B(A, int): var = 3 """B.var docstring""" - squere = ReadOnlyValueDescrpitor() - """Squere of variable `var`""" + square = ReadOnlyValueDescriptor() + """Square of variable `var`""" - squere_no_doc = ReadOnlyValueDescrpitor() # no doc-string + squere_no_doc = ReadOnlyValueDescriptor() # no doc-string def __init__(self, x, y, z, w): """__init__ docstring""" From 9637e7b4c4c27b9458da3e2d5e18c50396acdfb9 Mon Sep 17 00:00:00 2001 From: Kernc Date: Sun, 1 Sep 2019 16:07:07 +0200 Subject: [PATCH 6/7] update, simplify by moving Variable creation outside PEP 224 docstrings collection function. --- pdoc/__init__.py | 117 +++++++++++++----------------- pdoc/test/__init__.py | 15 ++-- pdoc/test/example_pkg/__init__.py | 27 ++++--- 3 files changed, 74 insertions(+), 85 deletions(-) diff --git a/pdoc/__init__.py b/pdoc/__init__.py index c10e9cdb..e2e42340 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -199,56 +199,46 @@ def _pairwise(iterable): return zip(a, b) -def _var_docstrings(doc_obj: Union['Module', 'Class'], *, - _init_tree: ast.FunctionDef = None) -> Dict[str, 'Variable']: +def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *, + _init_tree=None) -> Tuple[Dict[str, str], + Dict[str, str]]: """ - Extracts docstrings for variables of `doc_obj` + Extracts PEP-224 docstrings for variables of `doc_obj` (either a `pdoc.Module` or `pdoc.Class`). - Returns a dict mapping variable names to `pdoc.Variable` objects. - - For `pdoc.Class` objects, the dict contains class' instance - variables (defined as `self.something` in class' `__init__`), - recognized by `Variable.instance_var == True`. + Returns a tuple of two dicts mapping variable names to their docstrings. + The second dict contains instance variables and is non-empty only in case + `doc_obj` is a `pdoc.Class` which has `__init__` method. """ + # No variables in namespace packages + if isinstance(doc_obj, Module) and doc_obj.is_namespace: + return {}, {} + + vars = {} # type: Dict[str, str] + instance_vars = {} # type: Dict[str, str] + if _init_tree: - tree = _init_tree # type: Union[ast.Module, ast.FunctionDef] + tree = _init_tree else: - # No variables in namespace packages - if isinstance(doc_obj, Module) and doc_obj.is_namespace: - return {} try: tree = ast.parse(inspect.getsource(doc_obj.obj)) except (OSError, TypeError, SyntaxError): warn("Couldn't get/parse source of '{!r}'".format(doc_obj)) - return {} + return {}, {} + if isinstance(doc_obj, Class): tree = tree.body[0] # type: ignore # ast.parse creates a dummy ast.Module wrapper - vs = {} # type: Dict[str, Variable] - - cls = None - module = doc_obj - module_all = set(getattr(module.obj, '__all__', ())) - member_obj = dict(inspect.getmembers(doc_obj.obj)).get - - if isinstance(doc_obj, Class): - cls = doc_obj - module = doc_obj.module - - # For classes, first add instance variables defined in __init__ - if not _init_tree: - # Recursive call with just the __init__ tree + # For classes, maybe add instance variables defined in __init__ for node in tree.body: if isinstance(node, ast.FunctionDef) and node.name == '__init__': - vs.update(_var_docstrings(doc_obj, _init_tree=node)) + instance_vars, _ = _pep224_docstrings(doc_obj, _init_tree=node) break try: ast_AnnAssign = ast.AnnAssign # type: Type except AttributeError: # Python < 3.6 ast_AnnAssign = type(None) - ast_Assignments = (ast.Assign, ast_AnnAssign) for assign_node, str_node in _pairwise(ast.iter_child_nodes(tree)): @@ -275,20 +265,13 @@ def _var_docstrings(doc_obj: Union['Module', 'Class'], *, else: continue - if not _is_public(name): - continue - - if module_all and name not in module_all: - continue - docstring = inspect.cleandoc(str_node.value.s).strip() if not docstring: continue - vs[name] = Variable(name, module, docstring, - obj=member_obj(name), - cls=cls, instance_var=bool(_init_tree)) - return vs + vars[name] = docstring + + return vars, instance_vars def _is_public(ident_name): @@ -299,6 +282,10 @@ def _is_public(ident_name): return not ident_name.startswith("_") +def _is_function(obj): + return inspect.isroutine(obj) and callable(obj) + + def _filter_type(type: Type[T], values: Union[Iterable['Doc'], Dict[str, 'Doc']]) -> List[T]: """ @@ -538,6 +525,8 @@ def __init__(self, module: ModuleType, *, docfilter: Callable[[Doc], bool] = Non self._is_inheritance_linked = False """Re-entry guard for `pdoc.Module._link_inheritance()`.""" + var_docstrings, _ = _pep224_docstrings(self) + # Populate self.doc with this module's public members if hasattr(self.obj, '__all__'): public_objs = [] @@ -555,16 +544,17 @@ def is_from_this_module(obj): public_objs = [(name, inspect.unwrap(obj)) for name, obj in inspect.getmembers(self.obj) if (_is_public(name) and - is_from_this_module(obj))] + (is_from_this_module(obj) or name in var_docstrings))] index = list(self.obj.__dict__).index public_objs.sort(key=lambda i: index(i[0])) + for name, obj in public_objs: - if inspect.isroutine(obj): + if _is_function(obj): self.doc[name] = Function(name, self, obj) elif inspect.isclass(obj): self.doc[name] = Class(name, self, obj) - - self.doc.update(_var_docstrings(self)) + elif name in var_docstrings: + self.doc[name] = Variable(name, self, var_docstrings[name], obj=obj) # If the module is a package, scan the directory for submodules if self.is_package: @@ -799,8 +789,6 @@ def __init__(self, name, module, obj, *, docstring=None): self.doc = {} """A mapping from identifier name to a `pdoc.Doc` objects.""" - self.doc.update(_var_docstrings(self)) - public_objs = [(name, inspect.unwrap(obj)) for name, obj in inspect.getmembers(self.obj) # Filter only *own* members. The rest are inherited @@ -809,35 +797,30 @@ def __init__(self, name, module, obj, *, docstring=None): index = list(self.obj.__dict__).index public_objs.sort(key=lambda i: index(i[0])) + var_docstrings, instance_var_docstrings = _pep224_docstrings(self) + # Convert the public Python objects to documentation objects. for name, obj in public_objs: - if name in self.doc and self.doc[name].docstring: - if inspect.isroutine(obj) and not callable(obj): - assert isinstance(self.doc[name], Variable) - self.doc[name].instance_var = True - continue - if inspect.isroutine(obj) and callable(obj): + if _is_function(obj): self.doc[name] = Function( name, self.module, obj, cls=self, method=not self._method_type(self.obj, name)) - elif inspect.isroutine(obj): - self.doc[name] = Variable( - name, self.module, inspect.getdoc(obj), - obj=getattr(obj, '__get__', obj), - cls=self, instance_var=True) - elif (inspect.isdatadescriptor(obj) or - inspect.isgetsetdescriptor(obj) or - inspect.ismemberdescriptor(obj)): - self.doc[name] = Variable( - name, self.module, inspect.getdoc(obj), - obj=getattr(obj, 'fget', obj), - cls=self, instance_var=True) else: self.doc[name] = Variable( name, self.module, - docstring=isinstance(obj, type) and inspect.getdoc(obj) or "", - cls=self, - instance_var=name in getattr(self.obj, "__slots__", ())) + docstring=var_docstrings.get(name) or inspect.getdoc(obj), cls=self, + obj=getattr(obj, 'fget', getattr(obj, '__get__', obj)), + instance_var=(inspect.isdatadescriptor(obj) or + inspect.ismethoddescriptor(obj) or + inspect.isgetsetdescriptor(obj) or + inspect.ismemberdescriptor(obj) or + name in getattr(self.obj, '__slots__', ()))) + + for name, docstring in instance_var_docstrings.items(): + self.doc[name] = Variable( + name, self.module, docstring, cls=self, + obj=getattr(self.obj, name, None), + instance_var=True) @staticmethod def _method_type(cls: type, name: str): @@ -1027,7 +1010,7 @@ def __init__(self, name, module, obj, *, cls: Class = None, method=False): `method` should be `True` when the function is a method. In all other cases, it should be `False`. """ - assert callable(obj) + assert callable(obj), (name, module, obj) super().__init__(name, module, obj) self.cls = cls diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py index 7e046917..c615eceb 100644 --- a/pdoc/test/__init__.py +++ b/pdoc/test/__init__.py @@ -164,6 +164,7 @@ def test_html(self): exclude_patterns = [ ' class="ident">_private', ' class="ident">_Private', + 'non_callable_routine', ] package_files = { '': self.PUBLIC_FILES, @@ -298,6 +299,7 @@ def test_text(self): '_Private', 'subprocess', 'Hidden', + 'non_callable_routine', ] with self.subTest(package=EXAMPLE_MODULE): @@ -415,18 +417,19 @@ def test_instance_var(self): var = mod.doc['B'].doc['instance_var'] self.assertTrue(var.instance_var) - def test_readonly_var_descriptors(self): + def test_readonly_value_descriptors(self): pdoc.reset() mod = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE)) - var = mod.doc['B'].doc['square'] + var = mod.doc['B'].doc['ro_value_descriptor'] self.assertIsInstance(var, pdoc.Variable) self.assertTrue(var.instance_var) - self.assertEqual(var.docstring, """Square of variable `var`""") - var = mod.doc['B'].doc['squere_no_doc'] + self.assertEqual(var.docstring, """ro_value_descriptor docstring""") + self.assertTrue(var.source) + + var = mod.doc['B'].doc['ro_value_descriptor_no_doc'] self.assertIsInstance(var, pdoc.Variable) self.assertTrue(var.instance_var) - self.assertEqual(var.docstring, - """Read-only value descriptor, returns square of variable `var`""") + self.assertEqual(var.docstring, """Read-only value descriptor""") self.assertTrue(var.source) def test_builtin_methoddescriptors(self): diff --git a/pdoc/test/example_pkg/__init__.py b/pdoc/test/example_pkg/__init__.py index ea962ac8..56c7982f 100644 --- a/pdoc/test/example_pkg/__init__.py +++ b/pdoc/test/example_pkg/__init__.py @@ -21,15 +21,6 @@ def foo(env=os.environ): pass -class ReadOnlyValueDescriptor: - """Read-only value descriptor, returns square of variable `var`""" - - def __get__(self, instance, instance_type=None): - if instance is not None: - return instance.var ** 2 - return self - - class A: """`A` is base class for `example_pkg.B`.""" # Test refname link def overridden(self): @@ -42,6 +33,18 @@ def inherited(self): # Inherited in B """A.inherited docstring""" +non_callable_routine = staticmethod(lambda x: 2) # Not interpreted as Function; skipped + + +class ReadOnlyValueDescriptor: + """Read-only value descriptor""" + + def __get__(self, instance, instance_type=None): + if instance is not None: + return instance.var ** 2 + return self + + class B(A, int): """ B docstring @@ -55,10 +58,10 @@ class B(A, int): var = 3 """B.var docstring""" - square = ReadOnlyValueDescriptor() - """Square of variable `var`""" + ro_value_descriptor = ReadOnlyValueDescriptor() + """ro_value_descriptor docstring""" - squere_no_doc = ReadOnlyValueDescriptor() # no doc-string + ro_value_descriptor_no_doc = ReadOnlyValueDescriptor() # no doc-string def __init__(self, x, y, z, w): """`__init__` docstring""" From 215c3ce3710051c677af81b9986e97de4872b405 Mon Sep 17 00:00:00 2001 From: Kernc Date: Sun, 1 Sep 2019 16:14:11 +0200 Subject: [PATCH 7/7] silence mypy --- pdoc/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdoc/__init__.py b/pdoc/__init__.py index e2e42340..0a2021da 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -227,7 +227,7 @@ def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *, return {}, {} if isinstance(doc_obj, Class): - tree = tree.body[0] # type: ignore # ast.parse creates a dummy ast.Module wrapper + tree = tree.body[0] # ast.parse creates a dummy ast.Module wrapper # For classes, maybe add instance variables defined in __init__ for node in tree.body: