diff --git a/pdoc/__init__.py b/pdoc/__init__.py
index fe664565..4d5b4112 100644
--- a/pdoc/__init__.py
+++ b/pdoc/__init__.py
@@ -8,14 +8,15 @@
.. include:: ./documentation.md
"""
import ast
+import importlib.machinery
import importlib.util
import inspect
import os
import os.path as path
-import pkgutil
import re
import sys
import typing
+from contextlib import contextmanager
from copy import copy
from functools import lru_cache, reduce, partial
from itertools import tee, groupby
@@ -37,6 +38,8 @@
_URL_INDEX_MODULE_SUFFIX = '.m.html' # For modules named literal 'index'
_URL_PACKAGE_SUFFIX = '/index.html'
+_SOURCE_SUFFIXES = tuple(importlib.machinery.SOURCE_SUFFIXES)
+
T = TypeVar('T', bound='Doc')
__pdoc__ = {} # type: Dict[str, Union[bool, str]]
@@ -156,63 +159,32 @@ def import_module(module) -> ModuleType:
Return module object matching `module` specification (either a python
module path or a filesystem path to file/directory).
"""
- if isinstance(module, Module):
- module = module.module
- if isinstance(module, str):
+ @contextmanager
+ def _module_path(module):
+ from os.path import isfile, isdir, split, abspath, splitext
+ path, module = None, module
+ if isdir(module) or isfile(module) and module.endswith(_SOURCE_SUFFIXES):
+ path, module = split(splitext(abspath(module))[0])
try:
- module = importlib.import_module(module)
- except ImportError:
- pass
- except Exception as e:
- raise ImportError('Error importing {!r}: {}'.format(module, e))
-
- if inspect.ismodule(module):
- if module.__name__.startswith(__name__):
- # If this is pdoc itself, return without reloading.
- # Otherwise most `isinstance(..., pdoc.Doc)` calls won't
- # work correctly.
- return module
- return importlib.reload(module)
-
- # Try to load it as a filename
- if path.exists(module) and module.endswith('.py'):
- filename = module
- elif path.exists(module + '.py'):
- filename = module + '.py'
- elif path.exists(path.join(module, '__init__.py')):
- filename = path.join(module, '__init__.py')
- else:
- raise ValueError('File or module {!r} not found'.format(module))
-
- # If the path is relative, the whole of it is a python module path.
- # If the path is absolute, only the basename is.
- module_name = path.splitext(module)[0]
- if path.isabs(module):
- module_name = path.basename(module_name)
- else:
- module_name = path.splitdrive(module_name)[1]
- module_name = module_name.replace(path.sep, '.')
-
- spec = importlib.util.spec_from_file_location(module_name, path.abspath(filename))
- module = importlib.util.module_from_spec(spec)
- try:
- module.__loader__.exec_module(module)
- except Exception as e:
- raise ImportError('Error importing {!r}: {}'.format(filename, e))
-
- # For some reason, `importlib.util.module_from_spec` doesn't add
- # the module into `sys.modules`, and this later fails when
- # `inspect.getsource` tries to retrieve the module in AST parsing
- try:
- if sys.modules[module_name].__file__ != module.__file__:
- warn("Module {!r} in sys.modules loaded from {!r}. "
- "Now reloaded from {!r}.".format(module_name,
- sys.modules[module_name].__file__,
- module.__file__))
- except KeyError: # Module not yet in sys.modules
- pass
- sys.modules[module_name] = module
+ sys.path.insert(0, path)
+ yield module
+ finally:
+ sys.path.remove(path)
+ if isinstance(module, Module):
+ module = module.obj
+ if isinstance(module, str):
+ with _module_path(module) as module_path:
+ try:
+ module = importlib.import_module(module_path)
+ except Exception as e:
+ raise ImportError('Error importing {!r}: {}'.format(module, e))
+
+ assert inspect.ismodule(module)
+ # If this is pdoc itself, return without reloading. Otherwise later
+ # `isinstance(..., pdoc.Doc)` calls won't work correctly.
+ if not module.__name__.startswith(__name__):
+ module = importlib.reload(module)
return module
@@ -238,6 +210,9 @@ def _var_docstrings(doc_obj: Union['Module', 'Class'], *,
if _init_tree:
tree = _init_tree # type: Union[ast.Module, ast.FunctionDef]
else:
+ # No variables in namespace packages
+ if isinstance(doc_obj, Module) and doc_obj.is_namespace:
+ return {}
try:
tree = ast.parse(inspect.getsource(doc_obj.obj)) # type: ignore
except (OSError, TypeError, SyntaxError):
@@ -590,8 +565,22 @@ def is_from_this_module(obj):
# If the module is a package, scan the directory for submodules
if self.is_package:
- loc = getattr(self.module, "__path__", [path.dirname(self.obj.__file__)])
- for _, root, _ in pkgutil.iter_modules(loc):
+
+ def iter_modules(paths):
+ """
+ Custom implementation of `pkgutil.iter_modules()`
+ because that one doesn't play well with namespace packages.
+ See: https://github.com/pypa/setuptools/issues/83
+ """
+ from os.path import isdir, join, splitext
+ for pth in paths:
+ for file in os.listdir(pth):
+ if file.startswith(('.', '__pycache__', '__init__.py')):
+ continue
+ if file.endswith(_SOURCE_SUFFIXES) or isdir(join(pth, file)):
+ yield splitext(file)[0]
+
+ for root in iter_modules(self.obj.__path__):
# Ignore if this module was already doc'd.
if root in self.doc:
continue
@@ -708,6 +697,13 @@ def is_package(self):
"""
return hasattr(self.obj, "__path__")
+ @property
+ def is_namespace(self):
+ """
+ `True` if this module is a namespace package.
+ """
+ return self.obj.__spec__.origin in (None, 'namespace') # None in Py3.7+
+
def find_class(self, cls: type):
"""
Given a Python `cls` object, try to find it in this module
diff --git a/pdoc/templates/html.mako b/pdoc/templates/html.mako
index 8fc48d00..a9356d0a 100644
--- a/pdoc/templates/html.mako
+++ b/pdoc/templates/html.mako
@@ -108,7 +108,7 @@
% endfor
% endif
-
${module.name}
module
+ ${'Namespace' if module.is_namespace else 'Module'} ${module.name}
diff --git a/pdoc/templates/pdf.mako b/pdoc/templates/pdf.mako
index d2c94c0d..74f44053 100644
--- a/pdoc/templates/pdf.mako
+++ b/pdoc/templates/pdf.mako
@@ -66,7 +66,7 @@ links-as-notes: true
def to_md(text):
return _to_md(text, module)
%>
-${title(1, 'Module `%s`' % module.name, module.refname)}
+${title(1, ('Namespace' if module.is_namespace else 'Module') + ' `%s`' % module.name, module.refname)}
${module.docstring | to_md}
% if submodules:
diff --git a/pdoc/templates/text.mako b/pdoc/templates/text.mako
index 831aaf11..4dee1012 100644
--- a/pdoc/templates/text.mako
+++ b/pdoc/templates/text.mako
@@ -88,10 +88,11 @@ ${function(m) | indent}
classes = module.classes()
functions = module.functions()
submodules = module.submodules()
+ heading = 'Namespace' if module.is_namespace else 'Module'
%>
-Module ${module.name}
-=======${'=' * len(module.name)}
+${heading} ${module.name}
+=${'=' * (len(module.name) + len(heading))}
${module.docstring}
diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
index 790b2038..8ca3a13f 100644
--- a/pdoc/test/__init__.py
+++ b/pdoc/test/__init__.py
@@ -178,9 +178,9 @@ def test_html(self):
self._check_files(include_patterns, exclude_patterns)
filenames_files = {
- ('module.py',): [EXAMPLE_MODULE, EXAMPLE_MODULE + '/module.html'],
- ('module.py', 'subpkg2'): [f for f in self.PUBLIC_FILES
- if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE],
+ ('module.py',): ['module.html'],
+ ('module.py', 'subpkg2'): ['module.html', 'subpkg2',
+ 'subpkg2/index.html', 'subpkg2/module.html'],
}
with chdir(TESTS_BASEDIR):
for filenames, expected_files in filenames_files.items():
@@ -193,8 +193,7 @@ def test_html_multiple_files(self):
with chdir(TESTS_BASEDIR):
with run_html(EXAMPLE_MODULE + '/module.py', EXAMPLE_MODULE + '/subpkg2'):
self._basic_html_assertions(
- [f for f in self.PUBLIC_FILES
- if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE])
+ ['module.html', 'subpkg2', 'subpkg2/index.html', 'subpkg2/module.html'])
def test_html_identifier(self):
for package in ('', '._private'):
@@ -320,7 +319,7 @@ def test_text(self):
run(*(os.path.join(EXAMPLE_MODULE, f) for f in files))
out = stdout.getvalue()
for f in files:
- header = 'Module {}.{}'.format(EXAMPLE_MODULE, os.path.splitext(f)[0])
+ header = 'Module {}\n'.format(os.path.splitext(f)[0])
self.assertIn(header, out)
def test_text_identifier(self):
@@ -366,7 +365,7 @@ def setUp(self):
def test_module(self):
modules = {
EXAMPLE_MODULE: ('', ('index', 'module', 'subpkg', 'subpkg2')),
- os.path.join(EXAMPLE_MODULE, 'subpkg2'): ('.subpkg2', ('subpkg2.module',)),
+ EXAMPLE_MODULE + '.subpkg2': ('.subpkg2', ('subpkg2.module',)),
}
with chdir(TESTS_BASEDIR):
for module, (name_suffix, submodules) in modules.items():
@@ -378,11 +377,23 @@ def test_module(self):
[EXAMPLE_MODULE + '.' + m for m in submodules])
def test_import_filename(self):
- old_sys_path = sys.path.copy()
- sys.path.clear()
- with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+ with patch.object(sys, 'path', ['']), \
+ chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
pdoc.import_module('index')
- sys.path = old_sys_path
+
+ def test_imported_once(self):
+ with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+ pdoc.import_module('_imported_once.py')
+
+ def test_namespace(self):
+ # Test the three namespace types
+ # https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
+ for i in range(1, 4):
+ path = os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE, '_namespace', str(i))
+ with patch.object(sys, 'path', [os.path.join(path, 'a'),
+ os.path.join(path, 'b')]):
+ mod = pdoc.Module(pdoc.import_module('a.main'))
+ self.assertIn('D', mod.doc)
def test_module_allsubmodules(self):
m = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE + '._private'))
diff --git a/pdoc/test/example_pkg/_imported_once.py b/pdoc/test/example_pkg/_imported_once.py
new file mode 100644
index 00000000..c0f92c08
--- /dev/null
+++ b/pdoc/test/example_pkg/_imported_once.py
@@ -0,0 +1,8 @@
+import sys
+
+try:
+ sys._pdoc_imported_once_flag
+except AttributeError:
+ sys._pdoc_imported_once_flag = True
+else:
+ assert False, 'Module _imported_once already imported'
diff --git a/pdoc/test/example_pkg/_namespace/1/a/a/util.py b/pdoc/test/example_pkg/_namespace/1/a/a/util.py
new file mode 100644
index 00000000..646b07ae
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/1/a/a/util.py
@@ -0,0 +1,2 @@
+class C:
+ pass
diff --git a/pdoc/test/example_pkg/_namespace/1/b/a/main.py b/pdoc/test/example_pkg/_namespace/1/b/a/main.py
new file mode 100644
index 00000000..17c93e42
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/1/b/a/main.py
@@ -0,0 +1,5 @@
+from a.util import C
+
+
+class D(C):
+ pass
diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py
new file mode 100644
index 00000000..69e3be50
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py
@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/util.py b/pdoc/test/example_pkg/_namespace/2/a/a/util.py
new file mode 120000
index 00000000..4801dbe7
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/a/a/util.py
@@ -0,0 +1 @@
+../../../1/a/a/util.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py
new file mode 120000
index 00000000..d5a8ffe9
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py
@@ -0,0 +1 @@
+../../a/a/__init__.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/main.py b/pdoc/test/example_pkg/_namespace/2/b/a/main.py
new file mode 120000
index 00000000..f7fd43aa
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/b/a/main.py
@@ -0,0 +1 @@
+../../../1/b/a/main.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py
new file mode 100644
index 00000000..de40ea7c
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/util.py b/pdoc/test/example_pkg/_namespace/3/a/a/util.py
new file mode 120000
index 00000000..4801dbe7
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/a/a/util.py
@@ -0,0 +1 @@
+../../../1/a/a/util.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py
new file mode 120000
index 00000000..d5a8ffe9
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py
@@ -0,0 +1 @@
+../../a/a/__init__.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/main.py b/pdoc/test/example_pkg/_namespace/3/b/a/main.py
new file mode 120000
index 00000000..f7fd43aa
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/b/a/main.py
@@ -0,0 +1 @@
+../../../1/b/a/main.py
\ No newline at end of file