refactor(parser): strategy pattern for documentation parsing

package-parser/.gitignore

@@ -25,6 +25,9 @@ dist/
 htmlcov/
 .coverage
 
+# Test __init__.py files
+tests/**/__init__.py
+
 # Output of this tool
 out/
 tmp/

package-parser/package_parser/cli/_run_annotations.py

@@ -1,10 +1,10 @@
 import json
 from pathlib import Path
 
-from package_parser.model.annotations import AnnotationStore
-from package_parser.model.api import API
-from package_parser.model.usages import UsageCountStore
 from package_parser.processing.annotations import generate_annotations
+from package_parser.processing.annotations.model import AnnotationStore
+from package_parser.processing.api.model import API
+from package_parser.processing.usages.model import UsageCountStore
 from package_parser.utils import ensure_file_exists
 
 

package-parser/package_parser/cli/_run_api.py

@@ -4,8 +4,8 @@
 
 from package_parser.cli._json_encoder import CustomEncoder
 from package_parser.cli._shared_constants import _API_KEY
-from package_parser.model.api import API
 from package_parser.processing.api import get_api
+from package_parser.processing.api.model import API
 from package_parser.processing.dependencies import get_dependencies
 from package_parser.utils import ensure_file_exists
 

package-parser/package_parser/processing/annotations/_generate_annotations.py

@@ -1,6 +1,3 @@
-from package_parser.model.annotations import AnnotationStore
-from package_parser.model.api import API
-from package_parser.model.usages import UsageCountStore
 from package_parser.processing.annotations._generate_boundary_annotations import (
     _generate_boundary_annotations,
 )
@@ -16,6 +13,9 @@
 from package_parser.processing.annotations._usages_preprocessor import (
     _preprocess_usages,
 )
+from package_parser.processing.annotations.model import AnnotationStore
+from package_parser.processing.api.model import API
+from package_parser.processing.usages.model import UsageCountStore
 
 
 def generate_annotations(api: API, usages: UsageCountStore) -> AnnotationStore:

package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py

@@ -1,9 +1,9 @@
-from package_parser.model.annotations import (
+from package_parser.processing.annotations.model import (
     AnnotationStore,
     BoundaryAnnotation,
     Interval,
 )
-from package_parser.model.api import API
+from package_parser.processing.api.model import API
 
 from ._constants import autogen_author
 

package-parser/package_parser/processing/annotations/_generate_enum_annotations.py

@@ -1,7 +1,11 @@
 import re
 
-from package_parser.model.annotations import AnnotationStore, EnumAnnotation, EnumPair
-from package_parser.model.api import API
+from package_parser.processing.annotations.model import (
+    AnnotationStore,
+    EnumAnnotation,
+    EnumPair,
+)
+from package_parser.processing.api.model import API
 
 from ._constants import autogen_author
 

package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py

@@ -1,13 +1,13 @@
 from typing import Any, Optional
 
-from package_parser.model.annotations import (
+from package_parser.processing.annotations.model import (
     AnnotationStore,
     ConstantAnnotation,
     OptionalAnnotation,
     RequiredAnnotation,
 )
-from package_parser.model.api import API, Parameter
-from package_parser.model.usages import UsageCountStore
+from package_parser.processing.api.model import API, Parameter
+from package_parser.processing.usages.model import UsageCountStore
 
 from ._constants import autogen_author
 

package-parser/package_parser/processing/annotations/_generate_remove_annotations.py

@@ -1,6 +1,9 @@
-from package_parser.model.annotations import AnnotationStore, RemoveAnnotation
-from package_parser.model.api import API
-from package_parser.model.usages import UsageCountStore
+from package_parser.processing.annotations.model import (
+    AnnotationStore,
+    RemoveAnnotation,
+)
+from package_parser.processing.api.model import API
+from package_parser.processing.usages.model import UsageCountStore
 
 from ._constants import autogen_author
 

package-parser/package_parser/processing/annotations/_usages_preprocessor.py

@@ -1,5 +1,5 @@
-from package_parser.model.api import API
-from package_parser.model.usages import UsageCountStore
+from package_parser.processing.api.model import API
+from package_parser.processing.usages.model import UsageCountStore
 from package_parser.utils import parent_id
 
 

package-parser/package_parser/processing/api/_ast_visitor.py

@@ -1,30 +1,30 @@
-import inspect
 import logging
 import re
 from typing import Optional, Union
 
 import astroid
 from astroid.context import InferenceContext
 from astroid.helpers import safe_infer
-from numpydoc.docscrape import NumpyDocString
-from package_parser.model.api import (
+from package_parser.processing.api.model import (
     API,
     Class,
     FromImport,
     Function,
     Import,
     Module,
-    Parameter,
-    ParameterAndResultDocstring,
-    ParameterAssignment,
 )
 from package_parser.utils import parent_qualified_name
 
 from ._file_filters import _is_init_file
+from ._get_parameter_list import _get_parameter_list
+from .documentation import AbstractDocumentationParser
 
 
 class _AstVisitor:
-    def __init__(self, api: API) -> None:
+    def __init__(
+        self, documentation_parser: AbstractDocumentationParser, api: API
+    ) -> None:
+        self.documentation_parser: AbstractDocumentationParser = documentation_parser
         self.reexported: dict[str, list[str]] = {}
         self.api: API = api
         self.__declaration_stack: list[Union[Module, Class, Function]] = []
@@ -140,18 +140,15 @@ def enter_classdef(self, class_node: astroid.ClassDef) -> None:
         else:
             decorator_names = []
 
-        numpydoc = NumpyDocString(inspect.cleandoc(class_node.doc or ""))
-
         # Remember class, so we can later add methods
         class_ = Class(
-            self.__get_id(class_node.name),
-            qname,
-            decorator_names,
-            class_node.basenames,
-            self.is_public(class_node.name, qname),
-            self.reexported.get(qname, []),
-            _AstVisitor.__description(numpydoc),
-            class_node.doc,
+            id_=self.__get_id(class_node.name),
+            qname=qname,
+            decorators=decorator_names,
+            superclasses=class_node.basenames,
+            is_public=self.is_public(class_node.name, qname),
+            reexported_by=self.reexported.get(qname, []),
+            documentation=self.documentation_parser.get_class_documentation(class_node),
         )
         self.__declaration_stack.append(class_)
 
@@ -177,21 +174,25 @@ def enter_functiondef(self, function_node: astroid.FunctionDef) -> None:
         else:
             decorator_names = []
 
-        numpydoc = NumpyDocString(inspect.cleandoc(function_node.doc or ""))
         is_public = self.is_public(function_node.name, qname)
 
         function = Function(
-            self.__get_function_id(function_node.name, decorator_names),
-            qname,
-            decorator_names,
-            self.__function_parameters(
-                function_node, is_public, qname, self.__get_id(function_node.name)
+            id=self.__get_function_id(function_node.name, decorator_names),
+            qname=qname,
+            decorators=decorator_names,
+            parameters=_get_parameter_list(
+                self.documentation_parser,
+                function_node,
+                self.__get_id(function_node.name),
+                qname,
+                is_public,
+            ),
+            results=[],  # TODO: results
+            is_public=is_public,
+            reexported_by=self.reexported.get(qname, []),
+            documentation=self.documentation_parser.get_function_documentation(
+                function_node
             ),
-            [],  # TODO: results
-            is_public,
-            self.reexported.get(qname, []),
-            _AstVisitor.__description(numpydoc),
-            function_node.doc,
         )
         self.__declaration_stack.append(function)
 
@@ -211,122 +212,6 @@ def leave_functiondef(self, _: astroid.FunctionDef) -> None:
                 self.api.add_function(function)
                 parent.add_method(function.id)
 
-    @staticmethod
-    def __description(numpydoc: NumpyDocString) -> str:
-        has_summary = "Summary" in numpydoc and len(numpydoc["Summary"]) > 0
-        has_extended_summary = (
-            "Extended Summary" in numpydoc and len(numpydoc["Extended Summary"]) > 0
-        )
-
-        result = ""
-        if has_summary:
-            result += "\n".join(numpydoc["Summary"])
-        if has_summary and has_extended_summary:
-            result += "\n\n"
-        if has_extended_summary:
-            result += "\n".join(numpydoc["Extended Summary"])
-        return result
-
-    @staticmethod
-    def __function_parameters(
-        node: astroid.FunctionDef,
-        function_is_public: bool,
-        function_qname: str,
-        function_id: str,
-    ) -> list[Parameter]:
-        parameters = node.args
-        n_implicit_parameters = node.implicit_parameters()
-
-        # For constructors (__init__ functions) the parameters are described on the class
-        if node.name == "__init__" and isinstance(node.parent, astroid.ClassDef):
-            docstring = node.parent.doc
-        else:
-            docstring = node.doc
-        function_numpydoc = NumpyDocString(inspect.cleandoc(docstring or ""))
-
-        # Arguments that can be specified positionally only ( f(1) works but not f(x=1) )
-        result = [
-            Parameter(
-                id_=function_id + "/" + it.name,
-                name=it.name,
-                qname=function_qname + "." + it.name,
-                default_value=None,
-                assigned_by=ParameterAssignment.POSITION_ONLY,
-                is_public=function_is_public,
-                docstring=_AstVisitor.__parameter_docstring(function_numpydoc, it.name),
-            )
-            for it in parameters.posonlyargs
-        ]
-
-        # Arguments that can be specified positionally or by name ( f(1) and f(x=1) both work )
-        result += [
-            Parameter(
-                function_id + "/" + it.name,
-                it.name,
-                function_qname + "." + it.name,
-                _AstVisitor.__parameter_default(
-                    parameters.defaults,
-                    index - len(parameters.args) + len(parameters.defaults),
-                ),
-                ParameterAssignment.POSITION_OR_NAME,
-                function_is_public,
-                _AstVisitor.__parameter_docstring(function_numpydoc, it.name),
-            )
-            for index, it in enumerate(parameters.args)
-        ]
-
-        # Arguments that can be specified by name only ( f(x=1) works but not f(1) )
-        result += [
-            Parameter(
-                function_id + "/" + it.name,
-                it.name,
-                function_qname + "." + it.name,
-                _AstVisitor.__parameter_default(
-                    parameters.kw_defaults,
-                    index - len(parameters.kwonlyargs) + len(parameters.kw_defaults),
-                ),
-                ParameterAssignment.NAME_ONLY,
-                function_is_public,
-                _AstVisitor.__parameter_docstring(function_numpydoc, it.name),
-            )
-            for index, it in enumerate(parameters.kwonlyargs)
-        ]
-
-        implicit_parameters = result[:n_implicit_parameters]
-        for implicit_parameter in implicit_parameters:
-            implicit_parameter.assigned_by = ParameterAssignment.IMPLICIT
-
-        return result
-
-    @staticmethod
-    def __parameter_default(
-        defaults: list[astroid.NodeNG], default_index: int
-    ) -> Optional[str]:
-        if 0 <= default_index < len(defaults):
-            default = defaults[default_index]
-            if default is None:
-                return None
-            return default.as_string()
-        else:
-            return None
-
-    @staticmethod
-    def __parameter_docstring(
-        function_numpydoc: NumpyDocString, parameter_name: str
-    ) -> ParameterAndResultDocstring:
-        parameters_numpydoc = function_numpydoc["Parameters"]
-        candidate_parameters_numpydoc = [
-            it for it in parameters_numpydoc if it.name == parameter_name
-        ]
-
-        if len(candidate_parameters_numpydoc) > 0:
-            last_parameter_numpydoc = candidate_parameters_numpydoc[-1]
-            return ParameterAndResultDocstring(
-                last_parameter_numpydoc.type, "\n".join(last_parameter_numpydoc.desc)
-            )
-
-        return ParameterAndResultDocstring("", "")
-
     def is_public(self, name: str, qualified_name: str) -> bool:
         if name.startswith("_") and not name.endswith("__"):
             return False

package-parser/package_parser/processing/api/_get_api.py

@@ -3,7 +3,7 @@
 from typing import Optional
 
 import astroid
-from package_parser.model.api import API
+from package_parser.processing.api.model import API
 from package_parser.utils import ASTWalker
 
 from ._ast_visitor import _AstVisitor
@@ -14,6 +14,7 @@
     package_files,
     package_root,
 )
+from .documentation import NumpyDocParser
 
 
 def get_api(package_name: str, root: Optional[Path] = None) -> API:
@@ -24,7 +25,8 @@ def get_api(package_name: str, root: Optional[Path] = None) -> API:
     files = package_files(root)
 
     api = API(dist, package_name, dist_version)
-    callable_visitor = _AstVisitor(api)
+    documentation_parser = NumpyDocParser()
+    callable_visitor = _AstVisitor(documentation_parser, api)
     walker = ASTWalker(callable_visitor)
 
     for file in files: