Merge pull request #62 from kernc/new_import

Improved module importing / support for namespace packages
pdoc3 · May 3, 2019 · cbc829ba1f225d47c4bc98c34e117d43b64a40cb · cbc829b
2 parents 54fc6d0 + a3d0f25
commit cbc829ba1f225d47c4bc98c34e117d43b64a40cb
diff --git a/pdoc/__init__.py b/pdoc/__init__.py
@@ -8,14 +8,15 @@
 .. include:: ./documentation.md
 """
 import ast
+  
+import importlib.machinery
 import importlib.util
 import inspect
 import os
 import os.path as path
-  
-import pkgutil
 import re
 import sys
 import typing
+  
+from contextlib import contextmanager
 from copy import copy
 from functools import lru_cache, reduce, partial
 from itertools import tee, groupby
@@ -37,6 +38,8 @@
 _URL_INDEX_MODULE_SUFFIX = '.m.html'  # For modules named literal 'index'
 _URL_PACKAGE_SUFFIX = '/index.html'
+  
+_SOURCE_SUFFIXES = tuple(importlib.machinery.SOURCE_SUFFIXES)
+  
+
 T = TypeVar('T', bound='Doc')
 __pdoc__ = {}  # type: Dict[str, Union[bool, str]]
@@ -120,7 +123,7 @@ def _render_template(template_name, **kwargs):
        raise
-  
-def html(module_name, docfilter=None, **kwargs) -> str:
+  
+def html(module_name, docfilter=None, reload=False, **kwargs) -> str:
    """
    Returns the documentation for the module `module_name` in HTML
    format. The module must be a module or an importable string.
@@ -130,12 +133,12 @@ def html(module_name, docfilter=None, **kwargs) -> str:
    that takes a single argument (a documentation object) and returns
    `True` or `False`. If `False`, that object will not be documented.
    """
-  
-    mod = Module(import_module(module_name), docfilter=docfilter)
+  
+    mod = Module(import_module(module_name, reload=reload), docfilter=docfilter)
    link_inheritance()
    return mod.html(**kwargs)
-  
-def text(module_name, docfilter=None, **kwargs) -> str:
+  
+def text(module_name, docfilter=None, reload=False, **kwargs) -> str:
    """
    Returns the documentation for the module `module_name` in plain
    text format suitable for viewing on a terminal.
@@ -146,73 +149,42 @@ def text(module_name, docfilter=None, **kwargs) -> str:
    that takes a single argument (a documentation object) and returns
    `True` or `False`. If `False`, that object will not be documented.
    """
-  
-    mod = Module(import_module(module_name), docfilter=docfilter)
+  
+    mod = Module(import_module(module_name, reload=reload), docfilter=docfilter)
    link_inheritance()
    return mod.text(**kwargs)
-  
-def import_module(module) -> ModuleType:
+  
+def import_module(module, *, reload: bool = False) -> ModuleType:
    """
    Return module object matching `module` specification (either a python
    module path or a filesystem path to file/directory).
    """
-  
-    if isinstance(module, Module):
-  
-        module = module.module
-  
-    if isinstance(module, str):
+  
+    @contextmanager
+  
+    def _module_path(module):
+  
+        from os.path import isfile, isdir, split, abspath, splitext
+  
+        path, module = '_pdoc_dummy_nonexistent', module
+  
+        if isdir(module) or isfile(module) and module.endswith(_SOURCE_SUFFIXES):
+  
+            path, module = split(splitext(abspath(module))[0])
        try:
-  
-            module = importlib.import_module(module)
-  
-        except ImportError:
-  
-            pass
-  
-        except Exception as e:
-  
-            raise ImportError('Error importing {!r}: {}'.format(module, e))
-  
-
-  
-    if inspect.ismodule(module):
-  
-        if module.__name__.startswith(__name__):
-  
-            # If this is pdoc itself, return without reloading.
-  
-            # Otherwise most `isinstance(..., pdoc.Doc)` calls won't
-  
-            # work correctly.
-  
-            return module
-  
-        return importlib.reload(module)
-  
-
-  
-    # Try to load it as a filename
-  
-    if path.exists(module) and module.endswith('.py'):
-  
-        filename = module
-  
-    elif path.exists(module + '.py'):
-  
-        filename = module + '.py'
-  
-    elif path.exists(path.join(module, '__init__.py')):
-  
-        filename = path.join(module, '__init__.py')
-  
-    else:
-  
-        raise ValueError('File or module {!r} not found'.format(module))
-  
-
-  
-    # If the path is relative, the whole of it is a python module path.
-  
-    # If the path is absolute, only the basename is.
-  
-    module_name = path.splitext(module)[0]
-  
-    if path.isabs(module):
-  
-        module_name = path.basename(module_name)
-  
-    else:
-  
-        module_name = path.splitdrive(module_name)[1]
-  
-    module_name = module_name.replace(path.sep, '.')
-  
-
-  
-    spec = importlib.util.spec_from_file_location(module_name, path.abspath(filename))
-  
-    module = importlib.util.module_from_spec(spec)
-  
-    try:
-  
-        module.__loader__.exec_module(module)
-  
-    except Exception as e:
-  
-        raise ImportError('Error importing {!r}: {}'.format(filename, e))
-  
-
-  
-    # For some reason, `importlib.util.module_from_spec` doesn't add
-  
-    # the module into `sys.modules`, and this later fails when
-  
-    # `inspect.getsource` tries to retrieve the module in AST parsing
-  
-    try:
-  
-        if sys.modules[module_name].__file__ != module.__file__:
-  
-            warn("Module {!r} in sys.modules loaded from {!r}. "
-  
-                 "Now reloaded from {!r}.".format(module_name,
-  
-                                                  sys.modules[module_name].__file__,
-  
-                                                  module.__file__))
-  
-    except KeyError:  # Module not yet in sys.modules
-  
-        pass
-  
-    sys.modules[module_name] = module
+  
+            sys.path.insert(0, path)
+  
+            yield module
+  
+        finally:
+  
+            sys.path.remove(path)
+  
+    if isinstance(module, Module):
+  
+        module = module.obj
+  
+    if isinstance(module, str):
+  
+        with _module_path(module) as module_path:
+  
+            try:
+  
+                module = importlib.import_module(module_path)
+  
+            except Exception as e:
+  
+                raise ImportError('Error importing {!r}: {}'.format(module, e))
+  
+
+  
+    assert inspect.ismodule(module)
+  
+    # If this is pdoc itself, return without reloading. Otherwise later
+  
+    # `isinstance(..., pdoc.Doc)` calls won't work correctly.
+  
+    if reload and not module.__name__.startswith(__name__):
+  
+        module = importlib.reload(module)
    return module
@@ -238,6 +210,9 @@ def _var_docstrings(doc_obj: Union['Module', 'Class'], *,
    if _init_tree:
        tree = _init_tree  # type: Union[ast.Module, ast.FunctionDef]
    else:
+  
+        # No variables in namespace packages
+  
+        if isinstance(doc_obj, Module) and doc_obj.is_namespace:
+  
+            return {}
        try:
            tree = ast.parse(inspect.getsource(doc_obj.obj))  # type: ignore
        except (OSError, TypeError, SyntaxError):
@@ -595,8 +570,22 @@ def is_from_this_module(obj):
        # If the module is a package, scan the directory for submodules
        if self.is_package:
-  
-            loc = getattr(self.module, "__path__", [path.dirname(self.obj.__file__)])
-  
-            for _, root, _ in pkgutil.iter_modules(loc):
+  
+
+  
+            def iter_modules(paths):
+  
+                """
+  
+                Custom implementation of `pkgutil.iter_modules()`
+  
+                because that one doesn't play well with namespace packages.
+  
+                See: https://github.com/pypa/setuptools/issues/83
+  
+                """
+  
+                from os.path import isdir, join, splitext
+  
+                for pth in paths:
+  
+                    for file in os.listdir(pth):
+  
+                        if file.startswith(('.', '__pycache__', '__init__.py')):
+  
+                            continue
+  
+                        if file.endswith(_SOURCE_SUFFIXES) or isdir(join(pth, file)):
+  
+                            yield splitext(file)[0]
+  
+
+  
+            for root in iter_modules(self.obj.__path__):
                # Ignore if this module was already doc'd.
                if root in self.doc:
                    continue
@@ -713,6 +702,13 @@ def is_package(self):
        """
        return hasattr(self.obj, "__path__")
+  
+    @property
+  
+    def is_namespace(self):
+  
+        """
+  
+        `True` if this module is a namespace package.
+  
+        """
+  
+        return self.obj.__spec__.origin in (None, 'namespace')  # None in Py3.7+
+  
+
    def find_class(self, cls: type):
        """
        Given a Python `cls` object, try to find it in this module

diff --git a/pdoc/cli.py b/pdoc/cli.py
@@ -167,7 +167,7 @@ def do_GET(self):
        importlib.invalidate_caches()
        code = 200
        if self.path == "/":
-  
-            modules = [pdoc.import_module(module)
+  
+            modules = [pdoc.import_module(module, reload=True)
                       for module in self.args.modules]
            modules = sorted((module.__name__, inspect.getdoc(module))
                             for module in modules)
@@ -236,7 +236,8 @@ def html(self):
        """
        # TODO: pass extra pdoc.html() params
        return pdoc.html(self.import_path_from_req_url,
-  
-                         http_server=True, external_links=True, **self.template_config)
+  
+                         reload=True, http_server=True, external_links=True,
+  
+                         **self.template_config)
    def resolve_ext(self, import_path):
        def exists(p):

diff --git a/pdoc/templates/html.mako b/pdoc/templates/html.mako
@@ -108,7 +108,7 @@
      % endfor
    </nav>
  % endif
-  
-  <h1 class="title"><code>${module.name}</code> module</h1>
+  
+  <h1 class="title">${'Namespace' if module.is_namespace else 'Module'} <code>${module.name}</code></h1>
  </header>
  <section id="section-intro">

diff --git a/pdoc/templates/pdf.mako b/pdoc/templates/pdf.mako
@@ -66,7 +66,7 @@ links-as-notes: true
    def to_md(text):
        return _to_md(text, module)
 %>
-  
-${title(1, 'Module `%s`' % module.name, module.refname)}
+  
+${title(1, ('Namespace' if module.is_namespace else 'Module') + ' `%s`' % module.name, module.refname)}
 ${module.docstring | to_md}
 % if submodules:

diff --git a/pdoc/templates/text.mako b/pdoc/templates/text.mako
@@ -88,10 +88,11 @@ ${function(m) | indent}
  classes = module.classes()
  functions = module.functions()
  submodules = module.submodules()
+  
+  heading = 'Namespace' if module.is_namespace else 'Module'
 %>
-  
-Module ${module.name}
-  
-=======${'=' * len(module.name)}
+  
+${heading} ${module.name}
+  
+=${'=' * (len(module.name) + len(heading))}
 ${module.docstring}

diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
@@ -178,9 +178,9 @@ def test_html(self):
                    self._check_files(include_patterns, exclude_patterns)
        filenames_files = {
-  
-            ('module.py',): [EXAMPLE_MODULE, EXAMPLE_MODULE + '/module.html'],
-  
-            ('module.py', 'subpkg2'): [f for f in self.PUBLIC_FILES
-  
-                                       if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE],
+  
+            ('module.py',): ['module.html'],
+  
+            ('module.py', 'subpkg2'): ['module.html', 'subpkg2',
+  
+                                       'subpkg2/index.html', 'subpkg2/module.html'],
        }
        with chdir(TESTS_BASEDIR):
            for filenames, expected_files in filenames_files.items():
@@ -193,8 +193,7 @@ def test_html_multiple_files(self):
        with chdir(TESTS_BASEDIR):
            with run_html(EXAMPLE_MODULE + '/module.py', EXAMPLE_MODULE + '/subpkg2'):
                self._basic_html_assertions(
-  
-                    [f for f in self.PUBLIC_FILES
-  
-                     if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE])
+  
+                    ['module.html', 'subpkg2', 'subpkg2/index.html', 'subpkg2/module.html'])
    def test_html_identifier(self):
        for package in ('', '._private'):
@@ -320,7 +319,7 @@ def test_text(self):
                        run(*(os.path.join(EXAMPLE_MODULE, f) for f in files))
                        out = stdout.getvalue()
                    for f in files:
-  
-                        header = 'Module {}.{}'.format(EXAMPLE_MODULE, os.path.splitext(f)[0])
+  
+                        header = 'Module {}\n'.format(os.path.splitext(f)[0])
                        self.assertIn(header, out)
    def test_text_identifier(self):
@@ -366,7 +365,7 @@ def setUp(self):
    def test_module(self):
        modules = {
            EXAMPLE_MODULE: ('', ('index', 'module', 'subpkg', 'subpkg2')),
-  
-            os.path.join(EXAMPLE_MODULE, 'subpkg2'): ('.subpkg2', ('subpkg2.module',)),
+  
+            EXAMPLE_MODULE + '.subpkg2': ('.subpkg2', ('subpkg2.module',)),
        }
        with chdir(TESTS_BASEDIR):
            for module, (name_suffix, submodules) in modules.items():
@@ -378,11 +377,23 @@ def test_module(self):
                                     [EXAMPLE_MODULE + '.' + m for m in submodules])
    def test_import_filename(self):
-  
-        old_sys_path = sys.path.copy()
-  
-        sys.path.clear()
-  
-        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+  
+        with patch.object(sys, 'path', ['']), \
+  
+             chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
            pdoc.import_module('index')
-  
-        sys.path = old_sys_path
+  
+
+  
+    def test_imported_once(self):
+  
+        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+  
+            pdoc.import_module('_imported_once.py')
+  
+
+  
+    def test_namespace(self):
+  
+        # Test the three namespace types
+  
+        # https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
+  
+        for i in range(1, 4):
+  
+            path = os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE, '_namespace', str(i))
+  
+            with patch.object(sys, 'path', [os.path.join(path, 'a'),
+  
+                                            os.path.join(path, 'b')]):
+  
+                mod = pdoc.Module(pdoc.import_module('a.main'))
+  
+                self.assertIn('D', mod.doc)
    def test_module_allsubmodules(self):
        m = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE + '._private'))

diff --git a/pdoc/test/example_pkg/_imported_once.py b/pdoc/test/example_pkg/_imported_once.py
@@ -0,0 +1,8 @@
+  
+import sys
+  
+
+  
+try:
+  
+    sys._pdoc_imported_once_flag
+  
+except AttributeError:
+  
+    sys._pdoc_imported_once_flag = True
+  
+else:
+  
+    assert False, 'Module _imported_once already imported'
diff --git a/pdoc/test/example_pkg/_namespace/1/a/a/util.py b/pdoc/test/example_pkg/_namespace/1/a/a/util.py
@@ -0,0 +1,2 @@
+  
+class C:
+  
+    pass
diff --git a/pdoc/test/example_pkg/_namespace/1/b/a/main.py b/pdoc/test/example_pkg/_namespace/1/b/a/main.py
@@ -0,0 +1,5 @@
+  
+from a.util import C
+  
+
+  
+
+  
+class D(C):
+  
+    pass
diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py
@@ -0,0 +1 @@
+  
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/util.py b/pdoc/test/example_pkg/_namespace/2/a/a/util.py
@@ -0,0 +1 @@
+  
+../../../1/a/a/util.py
diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py
@@ -0,0 +1 @@
+  
+../../a/a/__init__.py
diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/main.py b/pdoc/test/example_pkg/_namespace/2/b/a/main.py
@@ -0,0 +1 @@
+  
+../../../1/b/a/main.py
diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py
@@ -0,0 +1 @@
+  
+__import__('pkg_resources').declare_namespace(__name__)
diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/util.py b/pdoc/test/example_pkg/_namespace/3/a/a/util.py
@@ -0,0 +1 @@
+  
+../../../1/a/a/util.py
diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py
@@ -0,0 +1 @@
+  
+../../a/a/__init__.py
diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/main.py b/pdoc/test/example_pkg/_namespace/3/b/a/main.py
@@ -0,0 +1 @@
+  
+../../../1/b/a/main.py