API: New better module import / support for namespace packages

Breaks CLI behavior when modules specified as file paths:
Previously full relative path was considered module path,
now only the basename is.
Thus some tests break.

pdoc/__init__.py

@@ -8,14 +8,15 @@
.. include:: ./documentation.md
"""
import ast
import importlib.machinery
import importlib.util
import inspect
import os
import os.path as path
import pkgutil
import re
import sys
import typing
from contextlib import contextmanager
from copy import copy
from functools import lru_cache, reduce, partial
from itertools import tee, groupby
@@ -37,6 +38,8 @@
_URL_INDEX_MODULE_SUFFIX = '.m.html'  # For modules named literal 'index'
_URL_PACKAGE_SUFFIX = '/index.html'

_SOURCE_SUFFIXES = tuple(importlib.machinery.SOURCE_SUFFIXES)

T = TypeVar('T', bound='Doc')

__pdoc__ = {}  # type: Dict[str, Union[bool, str]]
@@ -156,63 +159,32 @@ def import_module(module) -> ModuleType:
    Return module object matching `module` specification (either a python
    module path or a filesystem path to file/directory).
    """
    if isinstance(module, Module):
        module = module.module
    if isinstance(module, str):
    @contextmanager
    def _module_path(module):
        from os.path import isfile, isdir, split, abspath, splitext
        path, module = None, module
        if isdir(module) or isfile(module) and module.endswith(_SOURCE_SUFFIXES):
            path, module = split(splitext(abspath(module))[0])
        try:
            module = importlib.import_module(module)
        except ImportError:
            pass
        except Exception as e:
            raise ImportError('Error importing {!r}: {}'.format(module, e))

    if inspect.ismodule(module):
        if module.__name__.startswith(__name__):
            # If this is pdoc itself, return without reloading.
            # Otherwise most `isinstance(..., pdoc.Doc)` calls won't
            # work correctly.
            return module
        return importlib.reload(module)

    # Try to load it as a filename
    if path.exists(module) and module.endswith('.py'):
        filename = module
    elif path.exists(module + '.py'):
        filename = module + '.py'
    elif path.exists(path.join(module, '__init__.py')):
        filename = path.join(module, '__init__.py')
    else:
        raise ValueError('File or module {!r} not found'.format(module))

    # If the path is relative, the whole of it is a python module path.
    # If the path is absolute, only the basename is.
    module_name = path.splitext(module)[0]
    if path.isabs(module):
        module_name = path.basename(module_name)
    else:
        module_name = path.splitdrive(module_name)[1]
    module_name = module_name.replace(path.sep, '.')

    spec = importlib.util.spec_from_file_location(module_name, path.abspath(filename))
    module = importlib.util.module_from_spec(spec)
    try:
        module.__loader__.exec_module(module)
    except Exception as e:
        raise ImportError('Error importing {!r}: {}'.format(filename, e))

    # For some reason, `importlib.util.module_from_spec` doesn't add
    # the module into `sys.modules`, and this later fails when
    # `inspect.getsource` tries to retrieve the module in AST parsing
    try:
        if sys.modules[module_name].__file__ != module.__file__:
            warn("Module {!r} in sys.modules loaded from {!r}. "
                 "Now reloaded from {!r}.".format(module_name,
                                                  sys.modules[module_name].__file__,
                                                  module.__file__))
    except KeyError:  # Module not yet in sys.modules
        pass
    sys.modules[module_name] = module
            sys.path.insert(0, path)
            yield module
        finally:
            sys.path.remove(path)

    if isinstance(module, Module):
        module = module.obj
    if isinstance(module, str):
        with _module_path(module) as module_path:
            try:
                module = importlib.import_module(module_path)
            except Exception as e:
                raise ImportError('Error importing {!r}: {}'.format(module, e))

    assert inspect.ismodule(module)
    # If this is pdoc itself, return without reloading. Otherwise later
    # `isinstance(..., pdoc.Doc)` calls won't work correctly.
    if not module.__name__.startswith(__name__):
        module = importlib.reload(module)
    return module


@@ -238,6 +210,9 @@ def _var_docstrings(doc_obj: Union['Module', 'Class'], *,
    if _init_tree:
        tree = _init_tree  # type: Union[ast.Module, ast.FunctionDef]
    else:
        # No variables in namespace packages
        if isinstance(doc_obj, Module) and doc_obj.is_namespace:
            return {}
        try:
            tree = ast.parse(inspect.getsource(doc_obj.obj))  # type: ignore
        except (OSError, TypeError, SyntaxError):
@@ -590,8 +565,22 @@ def is_from_this_module(obj):

        # If the module is a package, scan the directory for submodules
        if self.is_package:
            loc = getattr(self.module, "__path__", [path.dirname(self.obj.__file__)])
            for _, root, _ in pkgutil.iter_modules(loc):

            def iter_modules(paths):
                """
                Custom implementation of `pkgutil.iter_modules()`
                because that one doesn't play well with namespace packages.
                See: https://github.com/pypa/setuptools/issues/83
                """
                from os.path import isdir, join, splitext
                for pth in paths:
                    for file in os.listdir(pth):
                        if file.startswith(('.', '__pycache__', '__init__.py')):
                            continue
                        if file.endswith(_SOURCE_SUFFIXES) or isdir(join(pth, file)):
                            yield splitext(file)[0]

            for root in iter_modules(self.obj.__path__):
                # Ignore if this module was already doc'd.
                if root in self.doc:
                    continue
@@ -708,6 +697,13 @@ def is_package(self):
        """
        return hasattr(self.obj, "__path__")

    @property
    def is_namespace(self):
        """
        `True` if this module is a namespace package.
        """
        return self.obj.__spec__.origin in (None, 'namespace')  # None in Py3.7+

    def find_class(self, cls: type):
        """
        Given a Python `cls` object, try to find it in this module

pdoc/test/__init__.py

@@ -178,9 +178,9 @@ def test_html(self):
                    self._check_files(include_patterns, exclude_patterns)

        filenames_files = {
            ('module.py',): [EXAMPLE_MODULE, EXAMPLE_MODULE + '/module.html'],
            ('module.py', 'subpkg2'): [f for f in self.PUBLIC_FILES
                                       if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE],
            ('module.py',): ['module.html'],
            ('module.py', 'subpkg2'): ['module.html', 'subpkg2',
                                       'subpkg2/index.html', 'subpkg2/module.html'],
        }
        with chdir(TESTS_BASEDIR):
            for filenames, expected_files in filenames_files.items():
@@ -193,8 +193,7 @@ def test_html_multiple_files(self):
        with chdir(TESTS_BASEDIR):
            with run_html(EXAMPLE_MODULE + '/module.py', EXAMPLE_MODULE + '/subpkg2'):
                self._basic_html_assertions(
                    [f for f in self.PUBLIC_FILES
                     if 'module' in f or 'subpkg2' in f or f == EXAMPLE_MODULE])
                    ['module.html', 'subpkg2', 'subpkg2/index.html', 'subpkg2/module.html'])

    def test_html_identifier(self):
        for package in ('', '._private'):
@@ -320,7 +319,7 @@ def test_text(self):
                        run(*(os.path.join(EXAMPLE_MODULE, f) for f in files))
                        out = stdout.getvalue()
                    for f in files:
                        header = 'Module {}.{}'.format(EXAMPLE_MODULE, os.path.splitext(f)[0])
                        header = 'Module {}\n'.format(os.path.splitext(f)[0])
                        self.assertIn(header, out)

    def test_text_identifier(self):
@@ -366,7 +365,7 @@ def setUp(self):
    def test_module(self):
        modules = {
            EXAMPLE_MODULE: ('', ('index', 'module', 'subpkg', 'subpkg2')),
            os.path.join(EXAMPLE_MODULE, 'subpkg2'): ('.subpkg2', ('subpkg2.module',)),
            EXAMPLE_MODULE + '.subpkg2': ('.subpkg2', ('subpkg2.module',)),
        }
        with chdir(TESTS_BASEDIR):
            for module, (name_suffix, submodules) in modules.items():
@@ -378,11 +377,23 @@ def test_module(self):
                                     [EXAMPLE_MODULE + '.' + m for m in submodules])

    def test_import_filename(self):
        old_sys_path = sys.path.copy()
        sys.path.clear()
        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
        with patch.object(sys, 'path', ['']), \
             chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
            pdoc.import_module('index')
        sys.path = old_sys_path

    def test_imported_once(self):
        with chdir(os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE)):
            pdoc.import_module('_imported_once.py')

    def test_namespace(self):
        # Test the three namespace types
        # https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
        for i in range(1, 4):
            path = os.path.join(TESTS_BASEDIR, EXAMPLE_MODULE, '_namespace', str(i))
            with patch.object(sys, 'path', [os.path.join(path, 'a'),
                                            os.path.join(path, 'b')]):
                mod = pdoc.Module(pdoc.import_module('a.main'))
                self.assertIn('D', mod.doc)

    def test_module_allsubmodules(self):
        m = pdoc.Module(pdoc.import_module(EXAMPLE_MODULE + '._private'))

pdoc/test/example_pkg/_imported_once.py

@@ -0,0 +1,8 @@
import sys

try:
    sys._pdoc_imported_once_flag
except AttributeError:
    sys._pdoc_imported_once_flag = True
else:
    assert False, 'Module _imported_once already imported'

pdoc/test/example_pkg/_namespace/1/a/a/util.py

@@ -0,0 +1,2 @@
class C:
    pass

pdoc/test/example_pkg/_namespace/1/b/a/main.py

@@ -0,0 +1,5 @@
from a.util import C


class D(C):
    pass

pdoc/test/example_pkg/_namespace/2/a/a/__init__.py

@@ -0,0 +1 @@
__path__ = __import__('pkgutil').extend_path(__path__, __name__)

pdoc/test/example_pkg/_namespace/2/a/a/util.py

@@ -0,0 +1 @@
../../../1/a/a/util.py

pdoc/test/example_pkg/_namespace/2/b/a/__init__.py

@@ -0,0 +1 @@
../../a/a/__init__.py

pdoc/test/example_pkg/_namespace/2/b/a/main.py

@@ -0,0 +1 @@
../../../1/b/a/main.py

pdoc/test/example_pkg/_namespace/3/a/a/__init__.py

@@ -0,0 +1 @@
__import__('pkg_resources').declare_namespace(__name__)

pdoc/test/example_pkg/_namespace/3/a/a/util.py

@@ -0,0 +1 @@
../../../1/a/a/util.py

pdoc/test/example_pkg/_namespace/3/b/a/__init__.py

@@ -0,0 +1 @@
../../a/a/__init__.py

pdoc/test/example_pkg/_namespace/3/b/a/main.py

@@ -0,0 +1 @@
../../../1/b/a/main.py