diff --git a/pdoc/__init__.py b/pdoc/__init__.py index fe664565..4d5b4112 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -8,14 +8,15 @@ .. include:: ./documentation.md """ import ast +import importlib.machinery import importlib.util import inspect import os import os.path as path -import pkgutil import re import sys import typing +from contextlib import contextmanager from copy import copy from functools import lru_cache, reduce, partial from itertools import tee, groupby @@ -37,6 +38,8 @@ _URL_INDEX_MODULE_SUFFIX = '.m.html' # For modules named literal 'index' _URL_PACKAGE_SUFFIX = '/index.html' +_SOURCE_SUFFIXES = tuple(importlib.machinery.SOURCE_SUFFIXES) + T = TypeVar('T', bound='Doc') __pdoc__ = {} # type: Dict[str, Union[bool, str]] @@ -156,63 +159,32 @@ def import_module(module) -> ModuleType: Return module object matching `module` specification (either a python module path or a filesystem path to file/directory). """ - if isinstance(module, Module): - module = module.module - if isinstance(module, str): + @contextmanager + def _module_path(module): + from os.path import isfile, isdir, split, abspath, splitext + path, module = None, module + if isdir(module) or isfile(module) and module.endswith(_SOURCE_SUFFIXES): + path, module = split(splitext(abspath(module))[0]) try: - module = importlib.import_module(module) - except ImportError: - pass - except Exception as e: - raise ImportError('Error importing {!r}: {}'.format(module, e)) - - if inspect.ismodule(module): - if module.__name__.startswith(__name__): - # If this is pdoc itself, return without reloading. - # Otherwise most `isinstance(..., pdoc.Doc)` calls won't - # work correctly. - return module - return importlib.reload(module) - - # Try to load it as a filename - if path.exists(module) and module.endswith('.py'): - filename = module - elif path.exists(module + '.py'): - filename = module + '.py' - elif path.exists(path.join(module, '__init__.py')): - filename = path.join(module, '__init__.py') - else: - raise ValueError('File or module {!r} not found'.format(module)) - - # If the path is relative, the whole of it is a python module path. - # If the path is absolute, only the basename is. - module_name = path.splitext(module)[0] - if path.isabs(module): - module_name = path.basename(module_name) - else: - module_name = path.splitdrive(module_name)[1] - module_name = module_name.replace(path.sep, '.') - - spec = importlib.util.spec_from_file_location(module_name, path.abspath(filename)) - module = importlib.util.module_from_spec(spec) - try: - module.__loader__.exec_module(module) - except Exception as e: - raise ImportError('Error importing {!r}: {}'.format(filename, e)) - - # For some reason, `importlib.util.module_from_spec` doesn't add - # the module into `sys.modules`, and this later fails when - # `inspect.getsource` tries to retrieve the module in AST parsing - try: - if sys.modules[module_name].__file__ != module.__file__: - warn("Module {!r} in sys.modules loaded from {!r}. " - "Now reloaded from {!r}.".format(module_name, - sys.modules[module_name].__file__, - module.__file__)) - except KeyError: # Module not yet in sys.modules - pass - sys.modules[module_name] = module + sys.path.insert(0, path) + yield module + finally: + sys.path.remove(path) + if isinstance(module, Module): + module = module.obj + if isinstance(module, str): + with _module_path(module) as module_path: + try: + module = importlib.import_module(module_path) + except Exception as e: + raise ImportError('Error importing {!r}: {}'.format(module, e)) + + assert inspect.ismodule(module) + # If this is pdoc itself, return without reloading. Otherwise later + # `isinstance(..., pdoc.Doc)` calls won't work correctly. + if not module.__name__.startswith(__name__): + module = importlib.reload(module) return module @@ -238,6 +210,9 @@ def _var_docstrings(doc_obj: Union['Module', 'Class'], *, if _init_tree: tree = _init_tree # type: Union[ast.Module, ast.FunctionDef] else: + # No variables in namespace packages + if isinstance(doc_obj, Module) and doc_obj.is_namespace: + return {} try: tree = ast.parse(inspect.getsource(doc_obj.obj)) # type: ignore except (OSError, TypeError, SyntaxError): @@ -590,8 +565,22 @@ def is_from_this_module(obj): # If the module is a package, scan the directory for submodules if self.is_package: - loc = getattr(self.module, "__path__", [path.dirname(self.obj.__file__)]) - for _, root, _ in pkgutil.iter_modules(loc): + + def iter_modules(paths): + """ + Custom implementation of `pkgutil.iter_modules()` + because that one doesn't play well with namespace packages. + See: https://github.com/pypa/setuptools/issues/83 + """ + from os.path import isdir, join, splitext + for pth in paths: + for file in os.listdir(pth): + if file.startswith(('.', '__pycache__', '__init__.py')): + continue + if file.endswith(_SOURCE_SUFFIXES) or isdir(join(pth, file)): + yield splitext(file)[0] + + for root in iter_modules(self.obj.__path__): # Ignore if this module was already doc'd. if root in self.doc: continue @@ -708,6 +697,13 @@ def is_package(self): """ return hasattr(self.obj, "__path__") + @property + def is_namespace(self): + """ + `True` if this module is a namespace package. + """ + return self.obj.__spec__.origin in (None, 'namespace') # None in Py3.7+ + def find_class(self, cls: type): """ Given a Python `cls` object, try to find it in this module diff --git a/pdoc/templates/html.mako b/pdoc/templates/html.mako index 8fc48d00..a9356d0a 100644 --- a/pdoc/templates/html.mako +++ b/pdoc/templates/html.mako @@ -108,7 +108,7 @@ % endfor % endif -

${module.name} module

+

${'Namespace' if module.is_namespace else 'Module'} ${module.name}

diff --git a/pdoc/templates/pdf.mako b/pdoc/templates/pdf.mako index d2c94c0d..74f44053 100644 --- a/pdoc/templates/pdf.mako +++ b/pdoc/templates/pdf.mako @@ -66,7 +66,7 @@ links-as-notes: true def to_md(text): return _to_md(text, module) %> -${title(1, 'Module `%s`' % module.name, module.refname)} +${title(1, ('Namespace' if module.is_namespace else 'Module') + ' `%s`' % module.name, module.refname)} ${module.docstring | to_md} % if submodules: diff --git a/pdoc/templates/text.mako b/pdoc/templates/text.mako index 831aaf11..4dee1012 100644 --- a/pdoc/templates/text.mako +++ b/pdoc/templates/text.mako @@ -88,10 +88,11 @@ ${function(m) | indent} classes = module.classes() functions = module.functions() submodules = module.submodules() + heading = 'Namespace' if module.is_namespace else 'Module' %> -Module ${module.name} -=======${'=' * len(module.name)} +${heading} ${module.name} +=${'=' * (len(module.name) + len(heading))} ${module.docstring} diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py index 790b2038..8ca3a13f 100644 --- a/pdoc/test/__init__.py +++ b/pdoc/test/__init__.py @@ -178,9 +178,9 @@ def test_html(self): self._check_files(include_patterns, exclude_patterns) filenames_files = { - ('module.py',): [EXAMPLE_MODULE, EXAMPLE_MODULE + '/module.html'], - ('module.py', 'subpkg2'): [f for f in self.PUBLIC_FILES - if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE], + ('module.py',): ['module.html'], + ('module.py', 'subpkg2'): ['module.html', 'subpkg2', + 'subpkg2/index.html', 'subpkg2/module.html'], } with chdir(TESTS_BASEDIR): for filenames, expected_files in filenames_files.items(): @@ -193,8 +193,7 @@ def test_html_multiple_files(self): with chdir(TESTS_BASEDIR): with run_html(EXAMPLE_MODULE + '/module.py', EXAMPLE_MODULE + '/subpkg2'): self._basic_html_assertions( - [f for f in self.PUBLIC_FILES - if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE]) + ['module.html', 'subpkg2', 'subpkg2/index.html', 'subpkg2/module.html']) def test_html_identifier(self): for package in ('', '._private'): @@ -320,7 +319,7 @@ def test_text(self): run(*(os.path.join(EXAMPLE_MODULE, f) for f in files)) out = stdout.getvalue() for f in files: - header = 'Module {}.{}'.format(EXAMPLE_MODULE, os.path.splitext(f)[0]) + header = 'Module {}\n'.format(os.path.splitext(f)[0]) self.assertIn(header, out) def test_text_identifier(self): @@ -366,7 +365,7 @@ def setUp(self): def test_module(self): modules = { EXAMPLE_MODULE: ('', ('index', 'module', 'subpkg', 'subpkg2')), - os.path.join(EXAMPLE_MODULE, 'subpkg2'): ('.subpkg2', ('subpkg2.module',)), + EXAMPLE_MODULE + '.subpkg2': ('.subpkg2', ('subpkg2.module',)), } with chdir(TESTS_BASEDIR): for module, (name_suffix, submodules) in modules.items(): @@ -378,11 +377,23 @@ def test_module(self): [EXAMPLE_MODULE + '.' + m for m in submodules]) def test_import_filename(self): - old_sys_path = sys.path.copy() - sys.path.clear() - with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)): + with patch.object(sys, 'path', ['']), \ + chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)): pdoc.import_module('index') - sys.path = old_sys_path + + def test_imported_once(self): + with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)): + pdoc.import_module('_imported_once.py') + + def test_namespace(self): + # Test the three namespace types + # https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package + for i in range(1, 4): + path = os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE, '_namespace', str(i)) + with patch.object(sys, 'path', [os.path.join(path, 'a'), + os.path.join(path, 'b')]): + mod = pdoc.Module(pdoc.import_module('a.main')) + self.assertIn('D', mod.doc) def test_module_allsubmodules(self): m = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE + '._private')) diff --git a/pdoc/test/example_pkg/_imported_once.py b/pdoc/test/example_pkg/_imported_once.py new file mode 100644 index 00000000..c0f92c08 --- /dev/null +++ b/pdoc/test/example_pkg/_imported_once.py @@ -0,0 +1,8 @@ +import sys + +try: + sys._pdoc_imported_once_flag +except AttributeError: + sys._pdoc_imported_once_flag = True +else: + assert False, 'Module _imported_once already imported' diff --git a/pdoc/test/example_pkg/_namespace/1/a/a/util.py b/pdoc/test/example_pkg/_namespace/1/a/a/util.py new file mode 100644 index 00000000..646b07ae --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/1/a/a/util.py @@ -0,0 +1,2 @@ +class C: + pass diff --git a/pdoc/test/example_pkg/_namespace/1/b/a/main.py b/pdoc/test/example_pkg/_namespace/1/b/a/main.py new file mode 100644 index 00000000..17c93e42 --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/1/b/a/main.py @@ -0,0 +1,5 @@ +from a.util import C + + +class D(C): + pass diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py new file mode 100644 index 00000000..69e3be50 --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/util.py b/pdoc/test/example_pkg/_namespace/2/a/a/util.py new file mode 120000 index 00000000..4801dbe7 --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/2/a/a/util.py @@ -0,0 +1 @@ +../../../1/a/a/util.py \ No newline at end of file diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py new file mode 120000 index 00000000..d5a8ffe9 --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py @@ -0,0 +1 @@ +../../a/a/__init__.py \ No newline at end of file diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/main.py b/pdoc/test/example_pkg/_namespace/2/b/a/main.py new file mode 120000 index 00000000..f7fd43aa --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/2/b/a/main.py @@ -0,0 +1 @@ +../../../1/b/a/main.py \ No newline at end of file diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py new file mode 100644 index 00000000..de40ea7c --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/util.py b/pdoc/test/example_pkg/_namespace/3/a/a/util.py new file mode 120000 index 00000000..4801dbe7 --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/3/a/a/util.py @@ -0,0 +1 @@ +../../../1/a/a/util.py \ No newline at end of file diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py new file mode 120000 index 00000000..d5a8ffe9 --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py @@ -0,0 +1 @@ +../../a/a/__init__.py \ No newline at end of file diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/main.py b/pdoc/test/example_pkg/_namespace/3/b/a/main.py new file mode 120000 index 00000000..f7fd43aa --- /dev/null +++ b/pdoc/test/example_pkg/_namespace/3/b/a/main.py @@ -0,0 +1 @@ +../../../1/b/a/main.py \ No newline at end of file