Better error handling with ParseError.

Prepare for when we'll unstring the annotations.
Special case __all__ variable for their computed values is outputed.
Add tests infra for project-wide analysis.
Add a test for a project using wildcard imports.

docspec-python/pyproject.toml

@@ -18,12 +18,13 @@ python = "^3.7"
 docspec = "^2.2.1"
 "nr.util" = ">=0.7.0"
 black = "^23.1.0"
-libstatic = { git = "https://github.com/tristanlatr/libstatic", tag = "0.2.0.dev2", optional = true }
+beniget = { git = "https://github.com/tristanlatr/beniget", rev = "ca577df3cca73140d53a325624a0185735354b69" }
+libstatic = { git = "https://github.com/tristanlatr/libstatic", tag = "0.2.0.dev3", optional = true }
 ast_comments = { version = "^1.1.0", optional = true }
 astor = { version = ">=0.8.1", optional = true }
 
 [tool.poetry.extras]
-experimental = ["libstatic", "ast_comments", "astor"]
+experimental = ["beniget", "libstatic", "ast_comments", "astor"]
 
 [tool.poetry.dev-dependencies]
 black = "*"

docspec-python/src/docspec_python/parser2.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import ast
+from functools import partial
 import inspect
 import platform
 import sys
@@ -32,18 +33,34 @@ class ParserOptions:
     expand_names: bool = True
     builtins: bool = False
     dependencies: bool | int = False
+    verbosity:int = 0
     # python_version:tuple[int, int]
 
+class ParseError(Exception):
+    ...
 
 def parse_modules(modules: t.Sequence[ModSpec], options: ParserOptions | None = None) -> t.Iterator[docspec.Module]:
     options = options or ParserOptions()
-    proj = libstatic.Project(builtins=False, verbosity=-2)
+    proj = libstatic.Project(builtins=options.builtins, 
+                             verbosity=options.verbosity)
     initial_modules: dict[str, str] = {}  # libstatic may add the builtins module
     for src, modname, filename, is_package, is_stub in modules:
         initial_modules[modname] = src
-        proj.add_module(
-            ast.parse(src, filename=filename or "<unknown>"), modname, is_package=is_package, filename=filename
-        )
+        filename = filename or "<unknown>"
+        try:
+            node = ast.parse(src, filename=filename)
+        except SyntaxError as e:
+            raise ParseError(f'cannot parse file: {e}') from e
+        try:
+            proj.add_module(
+                node, 
+                modname, 
+                is_package=is_package, 
+                filename=filename
+            )
+        except libstatic.StaticException as e:
+            raise ParseError(f'cannot add module {modname!r} to the project: {e}') from e
+
     proj.analyze_project()
     parser = Parser(proj.state, options)
     for m in proj.state.get_all_modules():
@@ -160,19 +177,30 @@ def __init__(self, state: libstatic.State, options: ParserOptions) -> None:
         self.state = state
         self.options = options
 
-    def unparse(self, expr: ast.expr) -> str:
+    def unparse(self, expr: ast.expr, is_annotation: bool = True) -> str:
         nexpr = ast.Expr(expr)
         if not self.options.expand_names:
             return t.cast(str, unparse(nexpr).rstrip("\n"))
-        expand_expr = self.state.expand_expr
+        state = self.state
+        expand_expr = state.expand_expr
+        # expand_name = partial(state.expand_name, 
+        #                       scope=next(s for s in state.get_all_enclosing_scopes(expr) 
+        #                                  if not isinstance(s, libstatic.Func)), 
+        #                       is_annotation=True)
 
         class SourceGenerator(astor.SourceGenerator):  # type:ignore[misc]
             def visit_Name(self, node: ast.Name) -> None:
-                expanded = expand_expr(node)
-                if expanded:
+                expanded: str = expand_expr(node)
+                if expanded and not expanded.endswith('*'):
                     self.write(expanded)
-                else:
-                    self.write(node.id)
+                    return
+                # not needed until the parse support unstringed type annotations.
+                # elif is_annotation:
+                #     expanded = expand_name(node.id)
+                #     if expanded and not expanded.endswith('*'):
+                #         self.write(expanded)
+                #         return
+                self.write(node.id)
 
             def visit_Str(self, node: ast.Str) -> None:
                 # astor uses tripple quoted strings :/
@@ -205,11 +233,17 @@ def _yield_members(self, definition: libstatic.Def) -> t.Sequence[libstatic.Def]
         # to sort them by source code order here.
         state = self.state
         list_of_defs: list[list[libstatic.Def]] = []
-        for defs in state.get_locals(definition).values():
+        for name, defs in state.get_locals(definition).items():
             # they can be None values here :/
             defs = list(filter(None, defs))
             if not defs:
                 continue
+            if (name == '__all__' and isinstance(definition, libstatic.Mod) and
+                self.state.get_dunder_all(definition) is not None):
+                # take advantage of the fact the __all__ values are parsed
+                # by libstatic and output the computed value here, so we leave
+                # only one definition of __all__ here and special case-it later.
+                defs = [defs[-1]]
             list_of_defs.append(defs)
         # filter unreachable defs if it doesn't remove all
         # information we have about this symbol.
@@ -366,7 +400,7 @@ def _extract_metaclass(self, definition: libstatic.Cls) -> str | None:
         return None
 
     def _extract_return_type(self, returns: ast.expr | None) -> str | None:
-        return self.unparse(returns) if returns else None
+        return self.unparse(returns, is_annotation=True) if returns else None
 
     def _unparse_keywords(self, keywords: list[ast.keyword]) -> t.Iterable[str]:
         for n in keywords:
@@ -401,17 +435,23 @@ def _parse_argument(self, arg: ArgSpec) -> docspec.Argument:
             location=self._parse_location(self.state.get_def(arg.node)),
             name=arg.node.arg,
             type=arg.type,
-            datatype=self.unparse(arg.node.annotation) if arg.node.annotation else None,
+            datatype=self.unparse(arg.node.annotation, is_annotation=True) if arg.node.annotation else None,
             default_value=self.unparse(arg.default) if arg.default else None,
         )
 
     def _extract_variable_value_type(self, definition: libstatic.Def) -> tuple[str | None, str | None]:
+        # special-case __all__
+        scope = self.state.get_enclosing_scope(definition)
+        if definition.name() == '__all__' and isinstance(scope, libstatic.Mod):
+            computed_value = self.state.get_dunder_all(scope)
+            if computed_value is not None:
+                return (repr(computed_value), None)
         try:
             assign = self.state.get_parent_instance(definition.node, (ast.Assign, ast.AnnAssign))
         except libstatic.StaticException:
             return None, None
         if isinstance(assign, ast.AnnAssign):
-            return (self.unparse(assign.value) if assign.value else None, self.unparse(assign.annotation))
+            return (self.unparse(assign.value) if assign.value else None, self.unparse(assign.annotation, is_annotation=True))
         try:
             value = get_stored_value(definition.node, assign)
         except libstatic.StaticException:
@@ -422,7 +462,7 @@ def _extract_variable_value_type(self, definition: libstatic.Def) -> tuple[str |
         if annotation is None:
             # because the code is unfinished, 'self.unparse(annotation)' will never run and mypy complains
             pass  # TODO: do basic type inference
-        return (self.unparse(value), self.unparse(annotation) if annotation else None)  # type:ignore
+        return (self.unparse(value), self.unparse(annotation, is_annotation=True) if annotation else None)  # type:ignore
 
     # @t.overload
     # def parse(self, definition: libstatic.Mod) -> docspec.Module:

docspec-python/test/test_parser2.py

@@ -0,0 +1,223 @@
+
+import ast
+import inspect
+import sys
+import types
+from functools import wraps
+from io import StringIO
+from json import dumps
+from textwrap import dedent
+from typing import Any, Callable, List, Optional, TypeVar, Iterable
+
+import pytest
+from docspec import (
+    ApiObject,
+    Argument,
+    Class,
+    Decoration,
+    Docstring,
+    Function,
+    HasLocation,
+    HasMembers,
+    Indirection,
+    Location,
+    Module,
+    Variable,
+    _ModuleMemberType,
+    dump_module,
+)
+
+from .test_parser import DocspecTest, mkfunc, unset_location
+
+try:
+    from docspec_python import parser2
+except ImportError:
+    parser2 = None
+
+loc = Location('<test>', 0, None)
+
+def _parse_doc(docstring:str) -> Iterable[parser2.ModSpec]:
+    """
+    format is 
+    '''
+    > {'modname':'test', }
+    import sys
+    import thing
+    > {'modname':'test2', }
+    from test import thing
+    '''
+    """
+    docstring = '\n'+inspect.cleandoc(docstring)
+    # separate modules
+    for p in docstring.split('\n>'):
+        if not p:
+            continue
+        try:
+            meta, *src = p.splitlines()
+        except ValueError as e:
+            raise ValueError(f'value is: {p!r}') from e
+        parsed_meta = ast.literal_eval(meta)
+        assert isinstance(parsed_meta, dict)
+        yield parser2.ModSpec(src='\n'.join(src), **parsed_meta)
+
+
+def docspec_test(parser_options: parser2.ParserOptions | None = None, 
+                 strip_locations: bool = True
+) -> Callable[[DocspecTest], Callable[[], None]]:
+    """
+    Decorator for docspec unit tests, parser2.
+    """
+
+    def decorator(func: DocspecTest) -> Callable[[], None]:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> None:
+
+            if parser2 is None:
+                return
+
+            # parse docstring into a series of modules
+            mods = list(_parse_doc(func.__doc__ or ""))
+            parsed_modules = list(parser2.parse_modules(mods, options=parser_options))
+
+            # run test
+            expected_modules = func(*args, **kwargs)
+
+            if strip_locations:
+                for parsed_module in parsed_modules:
+                    unset_location(parsed_module)
+                for reference_module in expected_modules:
+                    unset_location(reference_module)
+            assert dumps([dump_module(r) for r in expected_modules], indent=2) == dumps([dump_module(p) for p in parsed_modules], indent=2)
+
+        return wrapper
+
+    return decorator
+
+@docspec_test(strip_locations=True)
+def test_funcdef_annotation_expanded() -> List[_ModuleMemberType]:
+    """
+    > {'modname':'mod', 'is_package':True}
+    from ._impl import Cls
+    def a() -> Cls:
+        ...
+    > {'modname':'mod._impl'}
+    class Cls:
+        ...
+    """
+    return [
+        Module(
+            location=loc, 
+            name='mod', 
+            docstring=None, 
+            members=[
+                Indirection(
+                    name='Cls',
+                    target='mod._impl.Cls',
+                    location=loc,
+                    docstring=None,
+                ),
+                Function(
+                    name="a",
+                    location=loc,
+                    docstring=None,
+                    modifiers=None,
+                    args=[],
+                    return_type='mod._impl.Cls',
+                    decorations=[],
+                )]),
+        Module(
+            location=loc, 
+            name='mod._impl', 
+            docstring=None, 
+            members=[
+                Class(
+                    name="Cls",
+                    location=loc,
+                    docstring=None,
+                    members=[],
+                    metaclass=None,
+                    bases=[],
+                    decorations=None,
+                )])
+    ]
+
+@docspec_test(strip_locations=True, parser_options=parser2.ParserOptions(verbosity=2))
+def test_wildcard_imports() -> List[_ModuleMemberType]:
+    """
+    > {'modname':'mod', 'is_package':True}
+    from ._impl import *
+    from ._impl2 import *
+    from ._impl3 import *
+    from ._impl3 import __all__ as _all3
+    __all__ = ['Cls2', 'Cls1']
+    __all__ += _all3
+
+    def a(x:Cls2, y:Cls5) -> Cls1:
+        ...
+    > {'modname':'mod._impl'}
+    class Cls1:
+        ...
+    > {'modname':'mod._impl2'}
+    class Cls2:
+        ...
+    > {'modname':'mod._impl3'}
+    class Cls3:
+        ...
+    class Cls4:
+        ...
+    class Cls5:
+        ...
+    __all__ = ['Cls3', 'Cls5']
+    """
+    return [
+        Module(
+            location=loc, 
+            name='mod', 
+            docstring=None, 
+            members=[
+                Indirection(location=loc, name='*', docstring=None, target='mod._impl.*'),
+                Indirection(location=loc, name='Cls1', docstring=None, target='mod._impl.Cls1'),
+                Indirection(location=loc, name='*', docstring=None, target='mod._impl2.*'),
+                Indirection(location=loc, name='Cls2', docstring=None, target='mod._impl2.Cls2'),
+                Indirection(location=loc, name='*', docstring=None, target='mod._impl3.*'),
+                Indirection(location=loc, name='Cls3', docstring=None, target='mod._impl3.Cls3'),
+                Indirection(location=loc, name='Cls5', docstring=None, target='mod._impl3.Cls5'),
+                Indirection(location=loc, name='_all3', docstring=None, target='mod._impl3.__all__'),
+                Variable(location=loc, name='__all__', docstring=None, value="['Cls2', 'Cls1', 'Cls3', 'Cls5']"),
+                Function(location=loc, name='a', modifiers=None, args=[
+                    Argument(location=loc, name='x', type=Argument.Type.POSITIONAL, 
+                             datatype='mod._impl2.Cls2'), 
+                    Argument(location=loc, name='y', type=Argument.Type.POSITIONAL, 
+                             datatype='mod._impl3.Cls5'), 
+                ], return_type='mod._impl.Cls1', docstring=None, decorations=[]),
+            ]),
+        Module(
+            location=loc, 
+            name='mod._impl', 
+            docstring=None, 
+            members=[
+                Class(location=loc, name='Cls1', docstring=None,
+                      members=[], metaclass=None, bases=[], decorations=None),
+            ]),
+        Module(
+            location=loc, 
+            name='mod._impl2', 
+            docstring=None, 
+            members=[
+                Class(location=loc, name='Cls2', docstring=None,
+                      members=[], metaclass=None, bases=[], decorations=None),
+            ]),
+        Module(
+            location=loc, 
+            name='mod._impl3', 
+            docstring=None, 
+            members=[
+                Class(location=loc, name='Cls3', docstring=None,
+                      members=[], metaclass=None, bases=[], decorations=None),
+                Class(location=loc, name='Cls4', docstring=None,
+                      members=[], metaclass=None, bases=[], decorations=None),
+                Class(location=loc, name='Cls5', docstring=None,
+                      members=[], metaclass=None, bases=[], decorations=None),
+                Variable(location=loc, name='__all__', docstring=None, value="['Cls3', 'Cls5']")
+            ])
+    ]