Merge pull request #62 from kernc/new_import

Improved module importing / support for namespace packages

pdoc/__init__.py

@@ -8,14 +8,15 @@
 .. include:: ./documentation.md
 """
 import ast
+import importlib.machinery
 import importlib.util
 import inspect
 import os
 import os.path as path
-import pkgutil
 import re
 import sys
 import typing
+from contextlib import contextmanager
 from copy import copy
 from functools import lru_cache, reduce, partial
 from itertools import tee, groupby
@@ -37,6 +38,8 @@
 _URL_INDEX_MODULE_SUFFIX = '.m.html'  # For modules named literal 'index'
 _URL_PACKAGE_SUFFIX = '/index.html'
 
+_SOURCE_SUFFIXES = tuple(importlib.machinery.SOURCE_SUFFIXES)
+
 T = TypeVar('T', bound='Doc')
 
 __pdoc__ = {}  # type: Dict[str, Union[bool, str]]
@@ -120,7 +123,7 @@ def _render_template(template_name, **kwargs):
         raise
 
 
-def html(module_name, docfilter=None, **kwargs) -> str:
+def html(module_name, docfilter=None, reload=False, **kwargs) -> str:
     """
     Returns the documentation for the module `module_name` in HTML
     format. The module must be a module or an importable string.
@@ -130,12 +133,12 @@ def html(module_name, docfilter=None, **kwargs) -> str:
     that takes a single argument (a documentation object) and returns
     `True` or `False`. If `False`, that object will not be documented.
     """
-    mod = Module(import_module(module_name), docfilter=docfilter)
+    mod = Module(import_module(module_name, reload=reload), docfilter=docfilter)
     link_inheritance()
     return mod.html(**kwargs)
 
 
-def text(module_name, docfilter=None, **kwargs) -> str:
+def text(module_name, docfilter=None, reload=False, **kwargs) -> str:
     """
     Returns the documentation for the module `module_name` in plain
     text format suitable for viewing on a terminal.
@@ -146,73 +149,42 @@ def text(module_name, docfilter=None, **kwargs) -> str:
     that takes a single argument (a documentation object) and returns
     `True` or `False`. If `False`, that object will not be documented.
     """
-    mod = Module(import_module(module_name), docfilter=docfilter)
+    mod = Module(import_module(module_name, reload=reload), docfilter=docfilter)
     link_inheritance()
     return mod.text(**kwargs)
 
 
-def import_module(module) -> ModuleType:
+def import_module(module, *, reload: bool = False) -> ModuleType:
     """
     Return module object matching `module` specification (either a python
     module path or a filesystem path to file/directory).
     """
-    if isinstance(module, Module):
-        module = module.module
-    if isinstance(module, str):
+    @contextmanager
+    def _module_path(module):
+        from os.path import isfile, isdir, split, abspath, splitext
+        path, module = '_pdoc_dummy_nonexistent', module
+        if isdir(module) or isfile(module) and module.endswith(_SOURCE_SUFFIXES):
+            path, module = split(splitext(abspath(module))[0])
         try:
-            module = importlib.import_module(module)
-        except ImportError:
-            pass
-        except Exception as e:
-            raise ImportError('Error importing {!r}: {}'.format(module, e))
-
-    if inspect.ismodule(module):
-        if module.__name__.startswith(__name__):
-            # If this is pdoc itself, return without reloading.
-            # Otherwise most `isinstance(..., pdoc.Doc)` calls won't
-            # work correctly.
-            return module
-        return importlib.reload(module)
-
-    # Try to load it as a filename
-    if path.exists(module) and module.endswith('.py'):
-        filename = module
-    elif path.exists(module + '.py'):
-        filename = module + '.py'
-    elif path.exists(path.join(module, '__init__.py')):
-        filename = path.join(module, '__init__.py')
-    else:
-        raise ValueError('File or module {!r} not found'.format(module))
-
-    # If the path is relative, the whole of it is a python module path.
-    # If the path is absolute, only the basename is.
-    module_name = path.splitext(module)[0]
-    if path.isabs(module):
-        module_name = path.basename(module_name)
-    else:
-        module_name = path.splitdrive(module_name)[1]
-    module_name = module_name.replace(path.sep, '.')
-
-    spec = importlib.util.spec_from_file_location(module_name, path.abspath(filename))
-    module = importlib.util.module_from_spec(spec)
-    try:
-        module.__loader__.exec_module(module)
-    except Exception as e:
-        raise ImportError('Error importing {!r}: {}'.format(filename, e))
-
-    # For some reason, `importlib.util.module_from_spec` doesn't add
-    # the module into `sys.modules`, and this later fails when
-    # `inspect.getsource` tries to retrieve the module in AST parsing
-    try:
-        if sys.modules[module_name].__file__ != module.__file__:
-            warn("Module {!r} in sys.modules loaded from {!r}. "
-                 "Now reloaded from {!r}.".format(module_name,
-                                                  sys.modules[module_name].__file__,
-                                                  module.__file__))
-    except KeyError:  # Module not yet in sys.modules
-        pass
-    sys.modules[module_name] = module
+            sys.path.insert(0, path)
+            yield module
+        finally:
+            sys.path.remove(path)
 
+    if isinstance(module, Module):
+        module = module.obj
+    if isinstance(module, str):
+        with _module_path(module) as module_path:
+            try:
+                module = importlib.import_module(module_path)
+            except Exception as e:
+                raise ImportError('Error importing {!r}: {}'.format(module, e))
+
+    assert inspect.ismodule(module)
+    # If this is pdoc itself, return without reloading. Otherwise later
+    # `isinstance(..., pdoc.Doc)` calls won't work correctly.
+    if reload and not module.__name__.startswith(__name__):
+        module = importlib.reload(module)
     return module
 
 
@@ -238,6 +210,9 @@ def _var_docstrings(doc_obj: Union['Module', 'Class'], *,
     if _init_tree:
         tree = _init_tree  # type: Union[ast.Module, ast.FunctionDef]
     else:
+        # No variables in namespace packages
+        if isinstance(doc_obj, Module) and doc_obj.is_namespace:
+            return {}
         try:
             tree = ast.parse(inspect.getsource(doc_obj.obj))  # type: ignore
         except (OSError, TypeError, SyntaxError):
@@ -595,8 +570,22 @@ def is_from_this_module(obj):
 
         # If the module is a package, scan the directory for submodules
         if self.is_package:
-            loc = getattr(self.module, "__path__", [path.dirname(self.obj.__file__)])
-            for _, root, _ in pkgutil.iter_modules(loc):
+
+            def iter_modules(paths):
+                """
+                Custom implementation of `pkgutil.iter_modules()`
+                because that one doesn't play well with namespace packages.
+                See: https://github.com/pypa/setuptools/issues/83
+                """
+                from os.path import isdir, join, splitext
+                for pth in paths:
+                    for file in os.listdir(pth):
+                        if file.startswith(('.', '__pycache__', '__init__.py')):
+                            continue
+                        if file.endswith(_SOURCE_SUFFIXES) or isdir(join(pth, file)):
+                            yield splitext(file)[0]
+
+            for root in iter_modules(self.obj.__path__):
                 # Ignore if this module was already doc'd.
                 if root in self.doc:
                     continue
@@ -713,6 +702,13 @@ def is_package(self):
         """
         return hasattr(self.obj, "__path__")
 
+    @property
+    def is_namespace(self):
+        """
+        `True` if this module is a namespace package.
+        """
+        return self.obj.__spec__.origin in (None, 'namespace')  # None in Py3.7+
+
     def find_class(self, cls: type):
         """
         Given a Python `cls` object, try to find it in this module

pdoc/cli.py

@@ -167,7 +167,7 @@ def do_GET(self):
         importlib.invalidate_caches()
         code = 200
         if self.path == "/":
-            modules = [pdoc.import_module(module)
+            modules = [pdoc.import_module(module, reload=True)
                        for module in self.args.modules]
             modules = sorted((module.__name__, inspect.getdoc(module))
                              for module in modules)
@@ -236,7 +236,8 @@ def html(self):
         """
         # TODO: pass extra pdoc.html() params
         return pdoc.html(self.import_path_from_req_url,
-                         http_server=True, external_links=True, **self.template_config)
+                         reload=True, http_server=True, external_links=True,
+                         **self.template_config)
 
     def resolve_ext(self, import_path):
         def exists(p):

pdoc/templates/html.mako

@@ -108,7 +108,7 @@
       % endfor
     </nav>
   % endif
-  <h1 class="title"><code>${module.name}</code> module</h1>
+  <h1 class="title">${'Namespace' if module.is_namespace else 'Module'} <code>${module.name}</code></h1>
   </header>
 
   <section id="section-intro">

pdoc/templates/pdf.mako

@@ -66,7 +66,7 @@ links-as-notes: true
     def to_md(text):
         return _to_md(text, module)
 %>
-${title(1, 'Module `%s`' % module.name, module.refname)}
+${title(1, ('Namespace' if module.is_namespace else 'Module') + ' `%s`' % module.name, module.refname)}
 ${module.docstring | to_md}
 
 % if submodules:

pdoc/templates/text.mako

@@ -88,10 +88,11 @@ ${function(m) | indent}
   classes = module.classes()
   functions = module.functions()
   submodules = module.submodules()
+  heading = 'Namespace' if module.is_namespace else 'Module'
 %>
 
-Module ${module.name}
-=======${'=' * len(module.name)}
+${heading} ${module.name}
+=${'=' * (len(module.name) + len(heading))}
 ${module.docstring}
 
 

pdoc/test/__init__.py

@@ -178,9 +178,9 @@ def test_html(self):
                     self._check_files(include_patterns, exclude_patterns)
 
         filenames_files = {
-            ('module.py',): [EXAMPLE_MODULE, EXAMPLE_MODULE + '/module.html'],
-            ('module.py', 'subpkg2'): [f for f in self.PUBLIC_FILES
-                                       if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE],
+            ('module.py',): ['module.html'],
+            ('module.py', 'subpkg2'): ['module.html', 'subpkg2',
+                                       'subpkg2/index.html', 'subpkg2/module.html'],
         }
         with chdir(TESTS_BASEDIR):
             for filenames, expected_files in filenames_files.items():
@@ -193,8 +193,7 @@ def test_html_multiple_files(self):
         with chdir(TESTS_BASEDIR):
             with run_html(EXAMPLE_MODULE + '/module.py', EXAMPLE_MODULE + '/subpkg2'):
                 self._basic_html_assertions(
-                    [f for f in self.PUBLIC_FILES
-                     if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE])
+                    ['module.html', 'subpkg2', 'subpkg2/index.html', 'subpkg2/module.html'])
 
     def test_html_identifier(self):
         for package in ('', '._private'):
@@ -320,7 +319,7 @@ def test_text(self):
                         run(*(os.path.join(EXAMPLE_MODULE, f) for f in files))
                         out = stdout.getvalue()
                     for f in files:
-                        header = 'Module {}.{}'.format(EXAMPLE_MODULE, os.path.splitext(f)[0])
+                        header = 'Module {}\n'.format(os.path.splitext(f)[0])
                         self.assertIn(header, out)
 
     def test_text_identifier(self):
@@ -366,7 +365,7 @@ def setUp(self):
     def test_module(self):
         modules = {
             EXAMPLE_MODULE: ('', ('index', 'module', 'subpkg', 'subpkg2')),
-            os.path.join(EXAMPLE_MODULE, 'subpkg2'): ('.subpkg2', ('subpkg2.module',)),
+            EXAMPLE_MODULE + '.subpkg2': ('.subpkg2', ('subpkg2.module',)),
         }
         with chdir(TESTS_BASEDIR):
             for module, (name_suffix, submodules) in modules.items():
@@ -378,11 +377,23 @@ def test_module(self):
                                      [EXAMPLE_MODULE + '.' + m for m in submodules])
 
     def test_import_filename(self):
-        old_sys_path = sys.path.copy()
-        sys.path.clear()
-        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+        with patch.object(sys, 'path', ['']), \
+             chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
             pdoc.import_module('index')
-        sys.path = old_sys_path
+
+    def test_imported_once(self):
+        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+            pdoc.import_module('_imported_once.py')
+
+    def test_namespace(self):
+        # Test the three namespace types
+        # https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
+        for i in range(1, 4):
+            path = os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE, '_namespace', str(i))
+            with patch.object(sys, 'path', [os.path.join(path, 'a'),
+                                            os.path.join(path, 'b')]):
+                mod = pdoc.Module(pdoc.import_module('a.main'))
+                self.assertIn('D', mod.doc)
 
     def test_module_allsubmodules(self):
         m = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE + '._private'))

pdoc/test/example_pkg/_imported_once.py

@@ -0,0 +1,8 @@
+import sys
+
+try:
+    sys._pdoc_imported_once_flag
+except AttributeError:
+    sys._pdoc_imported_once_flag = True
+else:
+    assert False, 'Module _imported_once already imported'

pdoc/test/example_pkg/_namespace/1/a/a/util.py

@@ -0,0 +1,2 @@
+class C:
+    pass

pdoc/test/example_pkg/_namespace/1/b/a/main.py

@@ -0,0 +1,5 @@
+from a.util import C
+
+
+class D(C):
+    pass

pdoc/test/example_pkg/_namespace/2/a/a/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

pdoc/test/example_pkg/_namespace/2/a/a/util.py

@@ -0,0 +1 @@
+../../../1/a/a/util.py

pdoc/test/example_pkg/_namespace/2/b/a/__init__.py

@@ -0,0 +1 @@
+../../a/a/__init__.py

pdoc/test/example_pkg/_namespace/2/b/a/main.py

@@ -0,0 +1 @@
+../../../1/b/a/main.py

pdoc/test/example_pkg/_namespace/3/a/a/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

pdoc/test/example_pkg/_namespace/3/a/a/util.py

@@ -0,0 +1 @@
+../../../1/a/a/util.py

pdoc/test/example_pkg/_namespace/3/b/a/__init__.py

@@ -0,0 +1 @@
+../../a/a/__init__.py

pdoc/test/example_pkg/_namespace/3/b/a/main.py

@@ -0,0 +1 @@
+../../../1/b/a/main.py