From facbb6f46db2475450f8f47fd0e7e59e625b77a0 Mon Sep 17 00:00:00 2001
From: Kernc <kerncece@gmail.com>
Date: Fri, 3 May 2019 05:02:26 +0200
Subject: [PATCH] API: New better module import / support for namespace
 packages

Breaks CLI behavior when modules specified as file paths:
Previously full relative path was considered module path,
now only the basename is.
Thus some tests break.
---
 pdoc/__init__.py                              | 112 +++++++++---------
 pdoc/templates/html.mako                      |   2 +-
 pdoc/templates/pdf.mako                       |   2 +-
 pdoc/templates/text.mako                      |   5 +-
 pdoc/test/__init__.py                         |  33 ++++--
 pdoc/test/example_pkg/_imported_once.py       |   8 ++
 .../test/example_pkg/_namespace/1/a/a/util.py |   2 +
 .../test/example_pkg/_namespace/1/b/a/main.py |   5 +
 .../example_pkg/_namespace/2/a/a/__init__.py  |   1 +
 .../test/example_pkg/_namespace/2/a/a/util.py |   1 +
 .../example_pkg/_namespace/2/b/a/__init__.py  |   1 +
 .../test/example_pkg/_namespace/2/b/a/main.py |   1 +
 .../example_pkg/_namespace/3/a/a/__init__.py  |   1 +
 .../test/example_pkg/_namespace/3/a/a/util.py |   1 +
 .../example_pkg/_namespace/3/b/a/__init__.py  |   1 +
 .../test/example_pkg/_namespace/3/b/a/main.py |   1 +
 16 files changed, 104 insertions(+), 73 deletions(-)
 create mode 100644 pdoc/test/example_pkg/_imported_once.py
 create mode 100644 pdoc/test/example_pkg/_namespace/1/a/a/util.py
 create mode 100644 pdoc/test/example_pkg/_namespace/1/b/a/main.py
 create mode 100644 pdoc/test/example_pkg/_namespace/2/a/a/__init__.py
 create mode 120000 pdoc/test/example_pkg/_namespace/2/a/a/util.py
 create mode 120000 pdoc/test/example_pkg/_namespace/2/b/a/__init__.py
 create mode 120000 pdoc/test/example_pkg/_namespace/2/b/a/main.py
 create mode 100644 pdoc/test/example_pkg/_namespace/3/a/a/__init__.py
 create mode 120000 pdoc/test/example_pkg/_namespace/3/a/a/util.py
 create mode 120000 pdoc/test/example_pkg/_namespace/3/b/a/__init__.py
 create mode 120000 pdoc/test/example_pkg/_namespace/3/b/a/main.py

diff --git a/pdoc/__init__.py b/pdoc/__init__.py
index fe664565..4d5b4112 100644
--- a/pdoc/__init__.py
+++ b/pdoc/__init__.py
@@ -8,14 +8,15 @@
 .. include:: ./documentation.md
 """
 import ast
+import importlib.machinery
 import importlib.util
 import inspect
 import os
 import os.path as path
-import pkgutil
 import re
 import sys
 import typing
+from contextlib import contextmanager
 from copy import copy
 from functools import lru_cache, reduce, partial
 from itertools import tee, groupby
@@ -37,6 +38,8 @@
 _URL_INDEX_MODULE_SUFFIX = '.m.html'  # For modules named literal 'index'
 _URL_PACKAGE_SUFFIX = '/index.html'
 
+_SOURCE_SUFFIXES = tuple(importlib.machinery.SOURCE_SUFFIXES)
+
 T = TypeVar('T', bound='Doc')
 
 __pdoc__ = {}  # type: Dict[str, Union[bool, str]]
@@ -156,63 +159,32 @@ def import_module(module) -> ModuleType:
     Return module object matching `module` specification (either a python
     module path or a filesystem path to file/directory).
     """
-    if isinstance(module, Module):
-        module = module.module
-    if isinstance(module, str):
+    @contextmanager
+    def _module_path(module):
+        from os.path import isfile, isdir, split, abspath, splitext
+        path, module = None, module
+        if isdir(module) or isfile(module) and module.endswith(_SOURCE_SUFFIXES):
+            path, module = split(splitext(abspath(module))[0])
         try:
-            module = importlib.import_module(module)
-        except ImportError:
-            pass
-        except Exception as e:
-            raise ImportError('Error importing {!r}: {}'.format(module, e))
-
-    if inspect.ismodule(module):
-        if module.__name__.startswith(__name__):
-            # If this is pdoc itself, return without reloading.
-            # Otherwise most `isinstance(..., pdoc.Doc)` calls won't
-            # work correctly.
-            return module
-        return importlib.reload(module)
-
-    # Try to load it as a filename
-    if path.exists(module) and module.endswith('.py'):
-        filename = module
-    elif path.exists(module + '.py'):
-        filename = module + '.py'
-    elif path.exists(path.join(module, '__init__.py')):
-        filename = path.join(module, '__init__.py')
-    else:
-        raise ValueError('File or module {!r} not found'.format(module))
-
-    # If the path is relative, the whole of it is a python module path.
-    # If the path is absolute, only the basename is.
-    module_name = path.splitext(module)[0]
-    if path.isabs(module):
-        module_name = path.basename(module_name)
-    else:
-        module_name = path.splitdrive(module_name)[1]
-    module_name = module_name.replace(path.sep, '.')
-
-    spec = importlib.util.spec_from_file_location(module_name, path.abspath(filename))
-    module = importlib.util.module_from_spec(spec)
-    try:
-        module.__loader__.exec_module(module)
-    except Exception as e:
-        raise ImportError('Error importing {!r}: {}'.format(filename, e))
-
-    # For some reason, `importlib.util.module_from_spec` doesn't add
-    # the module into `sys.modules`, and this later fails when
-    # `inspect.getsource` tries to retrieve the module in AST parsing
-    try:
-        if sys.modules[module_name].__file__ != module.__file__:
-            warn("Module {!r} in sys.modules loaded from {!r}. "
-                 "Now reloaded from {!r}.".format(module_name,
-                                                  sys.modules[module_name].__file__,
-                                                  module.__file__))
-    except KeyError:  # Module not yet in sys.modules
-        pass
-    sys.modules[module_name] = module
+            sys.path.insert(0, path)
+            yield module
+        finally:
+            sys.path.remove(path)
 
+    if isinstance(module, Module):
+        module = module.obj
+    if isinstance(module, str):
+        with _module_path(module) as module_path:
+            try:
+                module = importlib.import_module(module_path)
+            except Exception as e:
+                raise ImportError('Error importing {!r}: {}'.format(module, e))
+
+    assert inspect.ismodule(module)
+    # If this is pdoc itself, return without reloading. Otherwise later
+    # `isinstance(..., pdoc.Doc)` calls won't work correctly.
+    if not module.__name__.startswith(__name__):
+        module = importlib.reload(module)
     return module
 
 
@@ -238,6 +210,9 @@ def _var_docstrings(doc_obj: Union['Module', 'Class'], *,
     if _init_tree:
         tree = _init_tree  # type: Union[ast.Module, ast.FunctionDef]
     else:
+        # No variables in namespace packages
+        if isinstance(doc_obj, Module) and doc_obj.is_namespace:
+            return {}
         try:
             tree = ast.parse(inspect.getsource(doc_obj.obj))  # type: ignore
         except (OSError, TypeError, SyntaxError):
@@ -590,8 +565,22 @@ def is_from_this_module(obj):
 
         # If the module is a package, scan the directory for submodules
         if self.is_package:
-            loc = getattr(self.module, "__path__", [path.dirname(self.obj.__file__)])
-            for _, root, _ in pkgutil.iter_modules(loc):
+
+            def iter_modules(paths):
+                """
+                Custom implementation of `pkgutil.iter_modules()`
+                because that one doesn't play well with namespace packages.
+                See: https://github.com/pypa/setuptools/issues/83
+                """
+                from os.path import isdir, join, splitext
+                for pth in paths:
+                    for file in os.listdir(pth):
+                        if file.startswith(('.', '__pycache__', '__init__.py')):
+                            continue
+                        if file.endswith(_SOURCE_SUFFIXES) or isdir(join(pth, file)):
+                            yield splitext(file)[0]
+
+            for root in iter_modules(self.obj.__path__):
                 # Ignore if this module was already doc'd.
                 if root in self.doc:
                     continue
@@ -708,6 +697,13 @@ def is_package(self):
         """
         return hasattr(self.obj, "__path__")
 
+    @property
+    def is_namespace(self):
+        """
+        `True` if this module is a namespace package.
+        """
+        return self.obj.__spec__.origin in (None, 'namespace')  # None in Py3.7+
+
     def find_class(self, cls: type):
         """
         Given a Python `cls` object, try to find it in this module
diff --git a/pdoc/templates/html.mako b/pdoc/templates/html.mako
index 8fc48d00..a9356d0a 100644
--- a/pdoc/templates/html.mako
+++ b/pdoc/templates/html.mako
@@ -108,7 +108,7 @@
       % endfor
     </nav>
   % endif
-  <h1 class="title"><code>${module.name}</code> module</h1>
+  <h1 class="title">${'Namespace' if module.is_namespace else 'Module'} <code>${module.name}</code></h1>
   </header>
 
   <section id="section-intro">
diff --git a/pdoc/templates/pdf.mako b/pdoc/templates/pdf.mako
index d2c94c0d..74f44053 100644
--- a/pdoc/templates/pdf.mako
+++ b/pdoc/templates/pdf.mako
@@ -66,7 +66,7 @@ links-as-notes: true
     def to_md(text):
         return _to_md(text, module)
 %>
-${title(1, 'Module `%s`' % module.name, module.refname)}
+${title(1, ('Namespace' if module.is_namespace else 'Module') + ' `%s`' % module.name, module.refname)}
 ${module.docstring | to_md}
 
 % if submodules:
diff --git a/pdoc/templates/text.mako b/pdoc/templates/text.mako
index 831aaf11..4dee1012 100644
--- a/pdoc/templates/text.mako
+++ b/pdoc/templates/text.mako
@@ -88,10 +88,11 @@ ${function(m) | indent}
   classes = module.classes()
   functions = module.functions()
   submodules = module.submodules()
+  heading = 'Namespace' if module.is_namespace else 'Module'
 %>
 
-Module ${module.name}
-=======${'=' * len(module.name)}
+${heading} ${module.name}
+=${'=' * (len(module.name) + len(heading))}
 ${module.docstring}
 
 
diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
index 790b2038..8ca3a13f 100644
--- a/pdoc/test/__init__.py
+++ b/pdoc/test/__init__.py
@@ -178,9 +178,9 @@ def test_html(self):
                     self._check_files(include_patterns, exclude_patterns)
 
         filenames_files = {
-            ('module.py',): [EXAMPLE_MODULE, EXAMPLE_MODULE + '/module.html'],
-            ('module.py', 'subpkg2'): [f for f in self.PUBLIC_FILES
-                                       if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE],
+            ('module.py',): ['module.html'],
+            ('module.py', 'subpkg2'): ['module.html', 'subpkg2',
+                                       'subpkg2/index.html', 'subpkg2/module.html'],
         }
         with chdir(TESTS_BASEDIR):
             for filenames, expected_files in filenames_files.items():
@@ -193,8 +193,7 @@ def test_html_multiple_files(self):
         with chdir(TESTS_BASEDIR):
             with run_html(EXAMPLE_MODULE + '/module.py', EXAMPLE_MODULE + '/subpkg2'):
                 self._basic_html_assertions(
-                    [f for f in self.PUBLIC_FILES
-                     if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE])
+                    ['module.html', 'subpkg2', 'subpkg2/index.html', 'subpkg2/module.html'])
 
     def test_html_identifier(self):
         for package in ('', '._private'):
@@ -320,7 +319,7 @@ def test_text(self):
                         run(*(os.path.join(EXAMPLE_MODULE, f) for f in files))
                         out = stdout.getvalue()
                     for f in files:
-                        header = 'Module {}.{}'.format(EXAMPLE_MODULE, os.path.splitext(f)[0])
+                        header = 'Module {}\n'.format(os.path.splitext(f)[0])
                         self.assertIn(header, out)
 
     def test_text_identifier(self):
@@ -366,7 +365,7 @@ def setUp(self):
     def test_module(self):
         modules = {
             EXAMPLE_MODULE: ('', ('index', 'module', 'subpkg', 'subpkg2')),
-            os.path.join(EXAMPLE_MODULE, 'subpkg2'): ('.subpkg2', ('subpkg2.module',)),
+            EXAMPLE_MODULE + '.subpkg2': ('.subpkg2', ('subpkg2.module',)),
         }
         with chdir(TESTS_BASEDIR):
             for module, (name_suffix, submodules) in modules.items():
@@ -378,11 +377,23 @@ def test_module(self):
                                      [EXAMPLE_MODULE + '.' + m for m in submodules])
 
     def test_import_filename(self):
-        old_sys_path = sys.path.copy()
-        sys.path.clear()
-        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+        with patch.object(sys, 'path', ['']), \
+             chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
             pdoc.import_module('index')
-        sys.path = old_sys_path
+
+    def test_imported_once(self):
+        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
+            pdoc.import_module('_imported_once.py')
+
+    def test_namespace(self):
+        # Test the three namespace types
+        # https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
+        for i in range(1, 4):
+            path = os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE, '_namespace', str(i))
+            with patch.object(sys, 'path', [os.path.join(path, 'a'),
+                                            os.path.join(path, 'b')]):
+                mod = pdoc.Module(pdoc.import_module('a.main'))
+                self.assertIn('D', mod.doc)
 
     def test_module_allsubmodules(self):
         m = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE + '._private'))
diff --git a/pdoc/test/example_pkg/_imported_once.py b/pdoc/test/example_pkg/_imported_once.py
new file mode 100644
index 00000000..c0f92c08
--- /dev/null
+++ b/pdoc/test/example_pkg/_imported_once.py
@@ -0,0 +1,8 @@
+import sys
+
+try:
+    sys._pdoc_imported_once_flag
+except AttributeError:
+    sys._pdoc_imported_once_flag = True
+else:
+    assert False, 'Module _imported_once already imported'
diff --git a/pdoc/test/example_pkg/_namespace/1/a/a/util.py b/pdoc/test/example_pkg/_namespace/1/a/a/util.py
new file mode 100644
index 00000000..646b07ae
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/1/a/a/util.py
@@ -0,0 +1,2 @@
+class C:
+    pass
diff --git a/pdoc/test/example_pkg/_namespace/1/b/a/main.py b/pdoc/test/example_pkg/_namespace/1/b/a/main.py
new file mode 100644
index 00000000..17c93e42
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/1/b/a/main.py
@@ -0,0 +1,5 @@
+from a.util import C
+
+
+class D(C):
+    pass
diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py
new file mode 100644
index 00000000..69e3be50
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/a/a/__init__.py
@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/pdoc/test/example_pkg/_namespace/2/a/a/util.py b/pdoc/test/example_pkg/_namespace/2/a/a/util.py
new file mode 120000
index 00000000..4801dbe7
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/a/a/util.py
@@ -0,0 +1 @@
+../../../1/a/a/util.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py
new file mode 120000
index 00000000..d5a8ffe9
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/b/a/__init__.py
@@ -0,0 +1 @@
+../../a/a/__init__.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/2/b/a/main.py b/pdoc/test/example_pkg/_namespace/2/b/a/main.py
new file mode 120000
index 00000000..f7fd43aa
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/2/b/a/main.py
@@ -0,0 +1 @@
+../../../1/b/a/main.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py
new file mode 100644
index 00000000..de40ea7c
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/a/a/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
diff --git a/pdoc/test/example_pkg/_namespace/3/a/a/util.py b/pdoc/test/example_pkg/_namespace/3/a/a/util.py
new file mode 120000
index 00000000..4801dbe7
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/a/a/util.py
@@ -0,0 +1 @@
+../../../1/a/a/util.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py b/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py
new file mode 120000
index 00000000..d5a8ffe9
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/b/a/__init__.py
@@ -0,0 +1 @@
+../../a/a/__init__.py
\ No newline at end of file
diff --git a/pdoc/test/example_pkg/_namespace/3/b/a/main.py b/pdoc/test/example_pkg/_namespace/3/b/a/main.py
new file mode 120000
index 00000000..f7fd43aa
--- /dev/null
+++ b/pdoc/test/example_pkg/_namespace/3/b/a/main.py
@@ -0,0 +1 @@
+../../../1/b/a/main.py
\ No newline at end of file