diff --git a/package-parser/.gitignore b/package-parser/.gitignore index 761e860a8..f304e6a07 100644 --- a/package-parser/.gitignore +++ b/package-parser/.gitignore @@ -25,6 +25,9 @@ dist/ htmlcov/ .coverage +# Test __init__.py files +tests/**/__init__.py + # Output of this tool out/ tmp/ diff --git a/package-parser/package_parser/cli/_run_annotations.py b/package-parser/package_parser/cli/_run_annotations.py index 6d6c9e62b..ecb6221c8 100644 --- a/package-parser/package_parser/cli/_run_annotations.py +++ b/package-parser/package_parser/cli/_run_annotations.py @@ -1,10 +1,10 @@ import json from pathlib import Path -from package_parser.model.annotations import AnnotationStore -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore from package_parser.processing.annotations import generate_annotations +from package_parser.processing.annotations.model import AnnotationStore +from package_parser.processing.api.model import API +from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/cli/_run_api.py b/package-parser/package_parser/cli/_run_api.py index cc962efcc..a8372d7e6 100644 --- a/package-parser/package_parser/cli/_run_api.py +++ b/package-parser/package_parser/cli/_run_api.py @@ -4,8 +4,8 @@ from package_parser.cli._json_encoder import CustomEncoder from package_parser.cli._shared_constants import _API_KEY -from package_parser.model.api import API from package_parser.processing.api import get_api +from package_parser.processing.api.model import API from package_parser.processing.dependencies import get_dependencies from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/model/__init__.py b/package-parser/package_parser/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/package_parser/model/api/__init__.py b/package-parser/package_parser/model/api/__init__.py deleted file mode 100644 index 32007cdd3..000000000 --- a/package-parser/package_parser/model/api/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from ._api import ( - API, - API_SCHEMA_VERSION, - Class, - FromImport, - Function, - Import, - Module, - Parameter, - ParameterAndResultDocstring, - ParameterAssignment, - Result, - Type, -) -from ._parameter_dependencies import ( - Action, - APIDependencies, - Condition, - Dependency, - ParameterHasValue, - ParameterIsIgnored, - ParameterIsIllegal, - ParameterIsNone, - RuntimeCondition, - StaticCondition, -) -from ._types import AbstractType, BoundaryType, EnumType, NamedType, UnionType diff --git a/package-parser/package_parser/processing/annotations/_generate_annotations.py b/package-parser/package_parser/processing/annotations/_generate_annotations.py index b401a6468..b7603854a 100644 --- a/package-parser/package_parser/processing/annotations/_generate_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_annotations.py @@ -1,6 +1,3 @@ -from package_parser.model.annotations import AnnotationStore -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore from package_parser.processing.annotations._generate_boundary_annotations import ( _generate_boundary_annotations, ) @@ -16,6 +13,9 @@ from package_parser.processing.annotations._usages_preprocessor import ( _preprocess_usages, ) +from package_parser.processing.annotations.model import AnnotationStore +from package_parser.processing.api.model import API +from package_parser.processing.usages.model import UsageCountStore def generate_annotations(api: API, usages: UsageCountStore) -> AnnotationStore: diff --git a/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py b/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py index f1118c50f..57b2f0d50 100644 --- a/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py @@ -1,9 +1,9 @@ -from package_parser.model.annotations import ( +from package_parser.processing.annotations.model import ( AnnotationStore, BoundaryAnnotation, Interval, ) -from package_parser.model.api import API +from package_parser.processing.api.model import API from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py index 52b44659e..d5ae11138 100644 --- a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py @@ -1,7 +1,11 @@ import re -from package_parser.model.annotations import AnnotationStore, EnumAnnotation, EnumPair -from package_parser.model.api import API +from package_parser.processing.annotations.model import ( + AnnotationStore, + EnumAnnotation, + EnumPair, +) +from package_parser.processing.api.model import API from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py b/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py index 500393d79..b26ef9bd9 100644 --- a/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py @@ -1,13 +1,13 @@ from typing import Any, Optional -from package_parser.model.annotations import ( +from package_parser.processing.annotations.model import ( AnnotationStore, ConstantAnnotation, OptionalAnnotation, RequiredAnnotation, ) -from package_parser.model.api import API, Parameter -from package_parser.model.usages import UsageCountStore +from package_parser.processing.api.model import API, Parameter +from package_parser.processing.usages.model import UsageCountStore from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py index 3c7da97a3..0217c6c35 100644 --- a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py @@ -1,6 +1,9 @@ -from package_parser.model.annotations import AnnotationStore, RemoveAnnotation -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore +from package_parser.processing.annotations.model import ( + AnnotationStore, + RemoveAnnotation, +) +from package_parser.processing.api.model import API +from package_parser.processing.usages.model import UsageCountStore from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_usages_preprocessor.py b/package-parser/package_parser/processing/annotations/_usages_preprocessor.py index 698e1b189..7b17f0b8e 100644 --- a/package-parser/package_parser/processing/annotations/_usages_preprocessor.py +++ b/package-parser/package_parser/processing/annotations/_usages_preprocessor.py @@ -1,5 +1,5 @@ -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore +from package_parser.processing.api.model import API +from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import parent_id diff --git a/package-parser/package_parser/model/annotations/__init__.py b/package-parser/package_parser/processing/annotations/model/__init__.py similarity index 100% rename from package-parser/package_parser/model/annotations/__init__.py rename to package-parser/package_parser/processing/annotations/model/__init__.py diff --git a/package-parser/package_parser/model/annotations/_annotations.py b/package-parser/package_parser/processing/annotations/model/_annotations.py similarity index 100% rename from package-parser/package_parser/model/annotations/_annotations.py rename to package-parser/package_parser/processing/annotations/model/_annotations.py diff --git a/package-parser/package_parser/processing/api/_ast_visitor.py b/package-parser/package_parser/processing/api/_ast_visitor.py index 856fa49a6..2b7358243 100644 --- a/package-parser/package_parser/processing/api/_ast_visitor.py +++ b/package-parser/package_parser/processing/api/_ast_visitor.py @@ -1,4 +1,3 @@ -import inspect import logging import re from typing import Optional, Union @@ -6,25 +5,26 @@ import astroid from astroid.context import InferenceContext from astroid.helpers import safe_infer -from numpydoc.docscrape import NumpyDocString -from package_parser.model.api import ( +from package_parser.processing.api.model import ( API, Class, FromImport, Function, Import, Module, - Parameter, - ParameterAndResultDocstring, - ParameterAssignment, ) from package_parser.utils import parent_qualified_name from ._file_filters import _is_init_file +from ._get_parameter_list import _get_parameter_list +from .documentation import AbstractDocumentationParser class _AstVisitor: - def __init__(self, api: API) -> None: + def __init__( + self, documentation_parser: AbstractDocumentationParser, api: API + ) -> None: + self.documentation_parser: AbstractDocumentationParser = documentation_parser self.reexported: dict[str, list[str]] = {} self.api: API = api self.__declaration_stack: list[Union[Module, Class, Function]] = [] @@ -140,18 +140,15 @@ def enter_classdef(self, class_node: astroid.ClassDef) -> None: else: decorator_names = [] - numpydoc = NumpyDocString(inspect.cleandoc(class_node.doc or "")) - # Remember class, so we can later add methods class_ = Class( - self.__get_id(class_node.name), - qname, - decorator_names, - class_node.basenames, - self.is_public(class_node.name, qname), - self.reexported.get(qname, []), - _AstVisitor.__description(numpydoc), - class_node.doc, + id_=self.__get_id(class_node.name), + qname=qname, + decorators=decorator_names, + superclasses=class_node.basenames, + is_public=self.is_public(class_node.name, qname), + reexported_by=self.reexported.get(qname, []), + documentation=self.documentation_parser.get_class_documentation(class_node), ) self.__declaration_stack.append(class_) @@ -177,21 +174,25 @@ def enter_functiondef(self, function_node: astroid.FunctionDef) -> None: else: decorator_names = [] - numpydoc = NumpyDocString(inspect.cleandoc(function_node.doc or "")) is_public = self.is_public(function_node.name, qname) function = Function( - self.__get_function_id(function_node.name, decorator_names), - qname, - decorator_names, - self.__function_parameters( - function_node, is_public, qname, self.__get_id(function_node.name) + id=self.__get_function_id(function_node.name, decorator_names), + qname=qname, + decorators=decorator_names, + parameters=_get_parameter_list( + self.documentation_parser, + function_node, + self.__get_id(function_node.name), + qname, + is_public, + ), + results=[], # TODO: results + is_public=is_public, + reexported_by=self.reexported.get(qname, []), + documentation=self.documentation_parser.get_function_documentation( + function_node ), - [], # TODO: results - is_public, - self.reexported.get(qname, []), - _AstVisitor.__description(numpydoc), - function_node.doc, ) self.__declaration_stack.append(function) @@ -211,122 +212,6 @@ def leave_functiondef(self, _: astroid.FunctionDef) -> None: self.api.add_function(function) parent.add_method(function.id) - @staticmethod - def __description(numpydoc: NumpyDocString) -> str: - has_summary = "Summary" in numpydoc and len(numpydoc["Summary"]) > 0 - has_extended_summary = ( - "Extended Summary" in numpydoc and len(numpydoc["Extended Summary"]) > 0 - ) - - result = "" - if has_summary: - result += "\n".join(numpydoc["Summary"]) - if has_summary and has_extended_summary: - result += "\n\n" - if has_extended_summary: - result += "\n".join(numpydoc["Extended Summary"]) - return result - - @staticmethod - def __function_parameters( - node: astroid.FunctionDef, - function_is_public: bool, - function_qname: str, - function_id: str, - ) -> list[Parameter]: - parameters = node.args - n_implicit_parameters = node.implicit_parameters() - - # For constructors (__init__ functions) the parameters are described on the class - if node.name == "__init__" and isinstance(node.parent, astroid.ClassDef): - docstring = node.parent.doc - else: - docstring = node.doc - function_numpydoc = NumpyDocString(inspect.cleandoc(docstring or "")) - - # Arguments that can be specified positionally only ( f(1) works but not f(x=1) ) - result = [ - Parameter( - id_=function_id + "/" + it.name, - name=it.name, - qname=function_qname + "." + it.name, - default_value=None, - assigned_by=ParameterAssignment.POSITION_ONLY, - is_public=function_is_public, - docstring=_AstVisitor.__parameter_docstring(function_numpydoc, it.name), - ) - for it in parameters.posonlyargs - ] - - # Arguments that can be specified positionally or by name ( f(1) and f(x=1) both work ) - result += [ - Parameter( - function_id + "/" + it.name, - it.name, - function_qname + "." + it.name, - _AstVisitor.__parameter_default( - parameters.defaults, - index - len(parameters.args) + len(parameters.defaults), - ), - ParameterAssignment.POSITION_OR_NAME, - function_is_public, - _AstVisitor.__parameter_docstring(function_numpydoc, it.name), - ) - for index, it in enumerate(parameters.args) - ] - - # Arguments that can be specified by name only ( f(x=1) works but not f(1) ) - result += [ - Parameter( - function_id + "/" + it.name, - it.name, - function_qname + "." + it.name, - _AstVisitor.__parameter_default( - parameters.kw_defaults, - index - len(parameters.kwonlyargs) + len(parameters.kw_defaults), - ), - ParameterAssignment.NAME_ONLY, - function_is_public, - _AstVisitor.__parameter_docstring(function_numpydoc, it.name), - ) - for index, it in enumerate(parameters.kwonlyargs) - ] - - implicit_parameters = result[:n_implicit_parameters] - for implicit_parameter in implicit_parameters: - implicit_parameter.assigned_by = ParameterAssignment.IMPLICIT - - return result - - @staticmethod - def __parameter_default( - defaults: list[astroid.NodeNG], default_index: int - ) -> Optional[str]: - if 0 <= default_index < len(defaults): - default = defaults[default_index] - if default is None: - return None - return default.as_string() - else: - return None - - @staticmethod - def __parameter_docstring( - function_numpydoc: NumpyDocString, parameter_name: str - ) -> ParameterAndResultDocstring: - parameters_numpydoc = function_numpydoc["Parameters"] - candidate_parameters_numpydoc = [ - it for it in parameters_numpydoc if it.name == parameter_name - ] - - if len(candidate_parameters_numpydoc) > 0: - last_parameter_numpydoc = candidate_parameters_numpydoc[-1] - return ParameterAndResultDocstring( - last_parameter_numpydoc.type, "\n".join(last_parameter_numpydoc.desc) - ) - - return ParameterAndResultDocstring("", "") - def is_public(self, name: str, qualified_name: str) -> bool: if name.startswith("_") and not name.endswith("__"): return False diff --git a/package-parser/package_parser/processing/api/_get_api.py b/package-parser/package_parser/processing/api/_get_api.py index f5c8c1393..5060fc6bf 100644 --- a/package-parser/package_parser/processing/api/_get_api.py +++ b/package-parser/package_parser/processing/api/_get_api.py @@ -3,7 +3,7 @@ from typing import Optional import astroid -from package_parser.model.api import API +from package_parser.processing.api.model import API from package_parser.utils import ASTWalker from ._ast_visitor import _AstVisitor @@ -14,6 +14,7 @@ package_files, package_root, ) +from .documentation import NumpyDocParser def get_api(package_name: str, root: Optional[Path] = None) -> API: @@ -24,7 +25,8 @@ def get_api(package_name: str, root: Optional[Path] = None) -> API: files = package_files(root) api = API(dist, package_name, dist_version) - callable_visitor = _AstVisitor(api) + documentation_parser = NumpyDocParser() + callable_visitor = _AstVisitor(documentation_parser, api) walker = ASTWalker(callable_visitor) for file in files: diff --git a/package-parser/package_parser/processing/api/_get_parameter_list.py b/package-parser/package_parser/processing/api/_get_parameter_list.py new file mode 100644 index 000000000..34bd1ddd3 --- /dev/null +++ b/package-parser/package_parser/processing/api/_get_parameter_list.py @@ -0,0 +1,88 @@ +from typing import Optional + +import astroid +from package_parser.processing.api.documentation import AbstractDocumentationParser +from package_parser.processing.api.model import Parameter, ParameterAssignment + + +def _get_parameter_list( + documentation_parser: AbstractDocumentationParser, + function_node: astroid.FunctionDef, + function_id: str, + function_qname: str, + function_is_public: bool, +) -> list[Parameter]: + parameters = function_node.args + n_implicit_parameters = function_node.implicit_parameters() + + # Arguments that can be specified positionally only ( f(1) works but not f(x=1) ) + result = [ + Parameter( + id_=function_id + "/" + it.name, + name=it.name, + qname=function_qname + "." + it.name, + default_value=None, + assigned_by=ParameterAssignment.POSITION_ONLY, + is_public=function_is_public, + documentation=documentation_parser.get_parameter_documentation( + function_node, it.name + ), + ) + for it in parameters.posonlyargs + ] + + # Arguments that can be specified positionally or by name ( f(1) and f(x=1) both work ) + result += [ + Parameter( + id_=function_id + "/" + it.name, + name=it.name, + qname=function_qname + "." + it.name, + default_value=_get_parameter_default( + parameters.defaults, + index - len(parameters.args) + len(parameters.defaults), + ), + assigned_by=ParameterAssignment.POSITION_OR_NAME, + is_public=function_is_public, + documentation=documentation_parser.get_parameter_documentation( + function_node, it.name + ), + ) + for index, it in enumerate(parameters.args) + ] + + # Arguments that can be specified by name only ( f(x=1) works but not f(1) ) + result += [ + Parameter( + id_=function_id + "/" + it.name, + name=it.name, + qname=function_qname + "." + it.name, + default_value=_get_parameter_default( + parameters.kw_defaults, + index - len(parameters.kwonlyargs) + len(parameters.kw_defaults), + ), + assigned_by=ParameterAssignment.NAME_ONLY, + is_public=function_is_public, + documentation=documentation_parser.get_parameter_documentation( + function_node, it.name + ), + ) + for index, it in enumerate(parameters.kwonlyargs) + ] + + implicit_parameters = result[:n_implicit_parameters] + for implicit_parameter in implicit_parameters: + implicit_parameter.assigned_by = ParameterAssignment.IMPLICIT + + return result + + +def _get_parameter_default( + defaults: list[astroid.NodeNG], default_index: int +) -> Optional[str]: + if 0 <= default_index < len(defaults): + default = defaults[default_index] + if default is None: + return None + return default.as_string() + else: + return None diff --git a/package-parser/package_parser/processing/api/documentation/_APIElementDocumentation.py b/package-parser/package_parser/processing/api/documentation/_APIElementDocumentation.py new file mode 100644 index 000000000..47103135a --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_APIElementDocumentation.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass + + +@dataclass +class ClassDocumentation: + description: str = "" + full_docstring: str = "" + + @staticmethod + def from_dict(d: dict) -> ClassDocumentation: + return ClassDocumentation(**d) + + def to_dict(self): + return dataclasses.asdict(self) + + +@dataclass +class FunctionDocumentation: + description: str = "" + full_docstring: str = "" + + @staticmethod + def from_dict(d: dict) -> FunctionDocumentation: + return FunctionDocumentation(**d) + + def to_dict(self): + return dataclasses.asdict(self) + + +@dataclass +class ParameterDocumentation: + type: str = "" + default_value: str = "" + description: str = "" + + @staticmethod + def from_dict(d: dict) -> ParameterDocumentation: + return ParameterDocumentation(**d) + + def to_dict(self): + return dataclasses.asdict(self) diff --git a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py new file mode 100644 index 000000000..8bf0cff66 --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod + +import astroid + +from ._APIElementDocumentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, +) + + +class AbstractDocumentationParser(ABC): + @abstractmethod + def get_class_documentation( + self, class_node: astroid.ClassDef + ) -> ClassDocumentation: + pass + + @abstractmethod + def get_function_documentation( + self, function_node: astroid.FunctionDef + ) -> FunctionDocumentation: + pass + + @abstractmethod + def get_parameter_documentation( + self, function_node: astroid.FunctionDef, parameter_name: str + ) -> ParameterDocumentation: + pass diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py new file mode 100644 index 000000000..2c014326c --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -0,0 +1,150 @@ +import re +from typing import Optional, Tuple + +import astroid +import numpydoc.docscrape +from numpydoc.docscrape import NumpyDocString + +from ._AbstractDocumentationParser import AbstractDocumentationParser +from ._APIElementDocumentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, +) +from ._get_full_docstring import get_full_docstring + + +class NumpyDocParser(AbstractDocumentationParser): + """ + Parses documentation in the NumpyDoc format. See https://numpydoc.readthedocs.io/en/latest/format.html for more + information. + + This class is not thread-safe. Each thread should create its own instance. + """ + + def __init__(self): + self.__cached_function_node: Optional[astroid.FunctionDef] = None + self.__cached_numpydoc_string: Optional[NumpyDocString] = None + + def get_class_documentation( + self, class_node: astroid.ClassDef + ) -> ClassDocumentation: + docstring = get_full_docstring(class_node) + + return ClassDocumentation( + description=_get_description(NumpyDocString(docstring)), + full_docstring=docstring, + ) + + def get_function_documentation( + self, function_node: astroid.FunctionDef + ) -> FunctionDocumentation: + docstring = get_full_docstring(function_node) + + return FunctionDocumentation( + description=_get_description( + self.__get_cached_function_numpydoc_string(function_node, docstring) + ), + full_docstring=docstring, + ) + + def get_parameter_documentation( + self, function_node: astroid.FunctionDef, parameter_name: str + ) -> ParameterDocumentation: + + # For constructors (__init__ functions) the parameters are described on the class + if function_node.name == "__init__" and isinstance( + function_node.parent, astroid.ClassDef + ): + docstring = get_full_docstring(function_node.parent) + else: + docstring = get_full_docstring(function_node) + + # Find matching parameter docstrings. Numpydoc allows multiple parameters to be documented at once. See + # https://numpydoc.readthedocs.io/en/latest/format.html#parameters for more information. + function_numpydoc = self.__get_cached_function_numpydoc_string( + function_node, docstring + ) + all_parameters_numpydoc: list[ + numpydoc.docscrape.Parameter + ] = function_numpydoc.get("Parameters", []) + matching_parameters_numpydoc = [ + it + for it in all_parameters_numpydoc + if _is_matching_parameter_numpydoc(it, parameter_name) + ] + + if len(matching_parameters_numpydoc) == 0: + return ParameterDocumentation(type="", default_value="", description="") + + last_parameter_numpydoc = matching_parameters_numpydoc[-1] + type_, default_value = _get_type_and_default_value(last_parameter_numpydoc) + return ParameterDocumentation( + type=type_, + default_value=default_value, + description="\n".join( + [line.strip() for line in last_parameter_numpydoc.desc] + ), + ) + + def __get_cached_function_numpydoc_string( + self, function_node: astroid.FunctionDef, docstring: str + ) -> NumpyDocString: + """ + Returns the NumpyDocString for the given function node. It is only recomputed when the function node differs + from the previous one that was passed to this function. This avoids reparsing the docstring for the function + itself and all of its parameters. + + On Lars's system this caused a significant performance improvement: Previously, 8.382s were spent inside the + function get_parameter_documentation when parsing sklearn. Afterwards, it was only 2.113s. + """ + + if self.__cached_function_node is not function_node: + self.__cached_function_node = function_node + self.__cached_numpydoc_string = NumpyDocString(docstring) + + return self.__cached_numpydoc_string + + +def _get_description(numpydoc_string: NumpyDocString) -> str: + """ + Returns the concatenated summary and extended summary parts of the given docstring or an empty string if these parts + are blank. + """ + + summary: list[str] = numpydoc_string.get("Summary", []) + extended_summary: list[str] = numpydoc_string.get("Extended Summary", []) + + result = "" + result += "\n".join([line.strip() for line in summary]) + result += "\n\n" + result += "\n".join([line.strip() for line in extended_summary]) + return result.strip() + + +def _is_matching_parameter_numpydoc( + parameter_numpydoc: numpydoc.docscrape.Parameter, parameter_name: str +) -> bool: + """ + Returns whether the given NumpyDoc applied to the parameter with the given name. + """ + + return any( + name.strip() == parameter_name for name in parameter_numpydoc.name.split(",") + ) + + +def _get_type_and_default_value( + parameter_numpydoc: numpydoc.docscrape.Parameter, +) -> Tuple[str, str]: + """ + Returns the type and default value for the given NumpyDoc. + """ + + type_ = parameter_numpydoc.type + parts = re.split(r",\s*optional|,\s*default\s*[:=]?", type_) + + if len(parts) != 2: + return type_.strip(), "" + + return parts[0].strip(), parts[1].strip() diff --git a/package-parser/package_parser/processing/api/documentation/__init__.py b/package-parser/package_parser/processing/api/documentation/__init__.py new file mode 100644 index 000000000..0342e961c --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/__init__.py @@ -0,0 +1,8 @@ +from ._AbstractDocumentationParser import AbstractDocumentationParser +from ._APIElementDocumentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, +) +from ._get_full_docstring import get_full_docstring +from ._NumpyDocParser import NumpyDocParser diff --git a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py new file mode 100644 index 000000000..72d783d78 --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py @@ -0,0 +1,18 @@ +import inspect +from typing import Union + +import astroid + + +def get_full_docstring( + declaration: Union[astroid.ClassDef, astroid.FunctionDef] +) -> str: + """ + Returns the full docstring of the given declaration or an empty string if no docstring is available. Indentation is + cleaned up. + """ + + doc_node = declaration.doc_node + if doc_node is None: + return "" + return inspect.cleandoc(doc_node.value) diff --git a/package-parser/package_parser/processing/api/model/__init__.py b/package-parser/package_parser/processing/api/model/__init__.py new file mode 100644 index 000000000..104a58da6 --- /dev/null +++ b/package-parser/package_parser/processing/api/model/__init__.py @@ -0,0 +1,15 @@ +from ._api import ( + API, + API_SCHEMA_VERSION, + Class, + FromImport, + Function, + Import, + Module, + Parameter, + ParameterAssignment, + Result, + ResultDocstring, + Type, +) +from ._types import AbstractType, BoundaryType, EnumType, NamedType, UnionType diff --git a/package-parser/package_parser/model/api/_api.py b/package-parser/package_parser/processing/api/model/_api.py similarity index 89% rename from package-parser/package_parser/model/api/_api.py rename to package-parser/package_parser/processing/api/model/_api.py index 97f5fab35..6e26bc47c 100644 --- a/package-parser/package_parser/model/api/_api.py +++ b/package-parser/package_parser/processing/api/model/_api.py @@ -5,15 +5,15 @@ from enum import Enum from typing import Any, Optional -from package_parser.model.api._types import ( - AbstractType, - BoundaryType, - EnumType, - NamedType, - UnionType, +from package_parser.processing.api.documentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, ) from package_parser.utils import parent_id +from ._types import AbstractType, BoundaryType, EnumType, NamedType, UnionType + API_SCHEMA_VERSION = 1 @@ -208,8 +208,10 @@ def from_json(json: Any) -> Class: json.get("superclasses", []), json.get("is_public", True), json.get("reexported_by", []), - json.get("description", ""), - json.get("docstring", ""), + ClassDocumentation( + description=json.get("description", ""), + full_docstring=json.get("docstring", ""), + ), ) for method_id in json["methods"]: @@ -225,8 +227,7 @@ def __init__( superclasses: list[str], is_public: bool, reexported_by: list[str], - description: str, - docstring: str, + documentation: ClassDocumentation, ) -> None: self.id: str = id_ self.qname: str = qname @@ -235,8 +236,7 @@ def __init__( self.methods: list[str] = [] self.is_public: bool = is_public self.reexported_by: list[str] = reexported_by - self.description: str = description - self.docstring: str = docstring + self.documentation: ClassDocumentation = documentation @property def name(self) -> str: @@ -255,8 +255,8 @@ def to_json(self) -> Any: "methods": self.methods, "is_public": self.is_public, "reexported_by": self.reexported_by, - "description": self.description, - "docstring": self.docstring, + "description": self.documentation.description, + "docstring": self.documentation.full_docstring, } @@ -269,8 +269,7 @@ class Function: results: list[Result] is_public: bool reexported_by: list[str] - description: str - docstring: str + documentation: FunctionDocumentation @staticmethod def from_json(json: Any) -> Function: @@ -285,8 +284,10 @@ def from_json(json: Any) -> Function: [Result.from_json(result_json) for result_json in json.get("results", [])], json.get("is_public", True), json.get("reexported_by", []), - json.get("description", ""), - json.get("docstring", ""), + FunctionDocumentation( + description=json.get("description", ""), + full_docstring=json.get("docstring", ""), + ), ) @property @@ -303,22 +304,20 @@ def to_json(self) -> Any: "results": [result.to_json() for result in self.results], "is_public": self.is_public, "reexported_by": self.reexported_by, - "description": self.description, - "docstring": self.docstring, + "description": self.documentation.description, + "docstring": self.documentation.full_docstring, } class Type: def __init__( self, - typestring: ParameterAndResultDocstring, + parameter_documentation: ParameterDocumentation, ) -> None: - self.type: Optional[AbstractType] = Type.create_type(typestring) + self.type: Optional[AbstractType] = Type.create_type(parameter_documentation) @classmethod - def create_type( - cls, docstring: ParameterAndResultDocstring - ) -> Optional[AbstractType]: + def create_type(cls, docstring: ParameterDocumentation) -> Optional[AbstractType]: type_string = docstring.type types: list[AbstractType] = list() @@ -402,7 +401,7 @@ def from_json(json: Any): json.get("default_value", None), ParameterAssignment[json.get("assigned_by", "POSITION_OR_NAME")], json.get("is_public", True), - ParameterAndResultDocstring.from_json(json.get("docstring", {})), + ParameterDocumentation.from_dict(json.get("docstring", {})), ) def __init__( @@ -413,7 +412,7 @@ def __init__( default_value: Optional[str], assigned_by: ParameterAssignment, is_public: bool, - docstring: ParameterAndResultDocstring, + documentation: ParameterDocumentation, ) -> None: self.id: str = id_ self.name: str = name @@ -421,8 +420,8 @@ def __init__( self.default_value: Optional[str] = default_value self.assigned_by: ParameterAssignment = assigned_by self.is_public: bool = is_public - self.docstring = docstring - self.type: Type = Type(docstring) + self.documentation = documentation + self.type: Type = Type(documentation) def is_optional(self) -> bool: return self.default_value is not None @@ -438,7 +437,7 @@ def to_json(self) -> Any: "default_value": self.default_value, "assigned_by": self.assigned_by.name, "is_public": self.is_public, - "docstring": self.docstring.to_json(), + "docstring": self.documentation.to_dict(), "type": self.type.to_json(), } @@ -453,13 +452,13 @@ class ParameterAssignment(Enum): @dataclass class Result: name: str - docstring: ParameterAndResultDocstring + docstring: ResultDocstring @staticmethod def from_json(json: Any) -> Result: return Result( json["name"], - ParameterAndResultDocstring.from_json(json.get("docstring", {})), + ResultDocstring.from_json(json.get("docstring", {})), ) def to_json(self) -> Any: @@ -467,13 +466,13 @@ def to_json(self) -> Any: @dataclass -class ParameterAndResultDocstring: +class ResultDocstring: type: str description: str @staticmethod def from_json(json: Any): - return ParameterAndResultDocstring( + return ResultDocstring( json.get("type", ""), json.get("description", ""), ) diff --git a/package-parser/package_parser/model/api/_types.py b/package-parser/package_parser/processing/api/model/_types.py similarity index 88% rename from package-parser/package_parser/model/api/_types.py rename to package-parser/package_parser/processing/api/model/_types.py index 266cec275..c2ac79cc9 100644 --- a/package-parser/package_parser/model/api/_types.py +++ b/package-parser/package_parser/processing/api/model/_types.py @@ -45,10 +45,10 @@ def remove_backslash(e: str): curr_quote = None for i, char in enumerate(enum_str): if char in quotes and (i == 0 or (i > 0 and enum_str[i - 1] != "\\")): - if inside_value == False: + if not inside_value: inside_value = True curr_quote = char - elif inside_value == True: + elif inside_value: if curr_quote == char: inside_value = False curr_quote = None @@ -76,7 +76,7 @@ class BoundaryType(AbstractType): INFINITY: ClassVar = "Infinity" base_type: str - min: Union[float, int] + min: Union[float, int, str] max: Union[float, int, str] min_inclusive: bool max_inclusive: bool @@ -92,6 +92,7 @@ def _is_inclusive(cls, bracket: str) -> bool: @classmethod def from_string(cls, string: str) -> Optional[BoundaryType]: + # language=PythonRegExp pattern = r"""(?Pfloat|int)?[ ] # optional base type of either float or int (in|of)[ ](the[ ])?(range|interval)[ ](of[ ])? # 'in' or 'of', optional 'the', 'range' or 'interval', optional 'of' `?(?P[\[(])(?P[-+]?\d+(.\d*)?|negative_infinity),[ ] # left side of the range @@ -102,17 +103,22 @@ def from_string(cls, string: str) -> Optional[BoundaryType]: base_type = match.group("base_type") if base_type is None: base_type = "float" - base_type = eval(base_type) - min_value = match.group("min") + min_value: Union[str, int, float] = match.group("min") if min_value != "negative_infinity": - min_value = base_type(min_value) + if base_type == "int": + min_value = int(min_value) + else: + min_value = float(min_value) else: min_value = BoundaryType.NEGATIVE_INFINITY - max_value = match.group("max") + max_value: Union[str, int, float] = match.group("max") if max_value != "infinity": - max_value = base_type(max_value) + if base_type == "int": + max_value = int(max_value) + else: + max_value = float(max_value) else: max_value = BoundaryType.INFINITY @@ -122,7 +128,7 @@ def from_string(cls, string: str) -> Optional[BoundaryType]: max_inclusive = BoundaryType._is_inclusive(max_bracket) return BoundaryType( - base_type.__name__, min_value, max_value, min_inclusive, max_inclusive + base_type, min_value, max_value, min_inclusive, max_inclusive ) return None diff --git a/package-parser/package_parser/processing/dependencies/__init__.py b/package-parser/package_parser/processing/dependencies/__init__.py index ab7fc933e..22deefff2 100644 --- a/package-parser/package_parser/processing/dependencies/__init__.py +++ b/package-parser/package_parser/processing/dependencies/__init__.py @@ -5,3 +5,16 @@ extract_lefts_and_rights, get_dependencies, ) +from ._parameter_dependencies import ( + Action, + Condition, + Dependency, + ParameterHasValue, + ParameterIsIgnored, + ParameterIsIllegal, + ParameterIsNone, + RuntimeAction, + RuntimeCondition, + StaticAction, + StaticCondition, +) diff --git a/package-parser/package_parser/processing/dependencies/_get_dependency.py b/package-parser/package_parser/processing/dependencies/_get_dependency.py index 08f4b4c22..84f3a6eea 100644 --- a/package-parser/package_parser/processing/dependencies/_get_dependency.py +++ b/package-parser/package_parser/processing/dependencies/_get_dependency.py @@ -1,24 +1,23 @@ from typing import Dict, List, Tuple, Union import spacy -from package_parser.model.api import ( - API, +from package_parser.processing.api.model import API, Parameter +from spacy.matcher import DependencyMatcher +from spacy.tokens import Token +from spacy.tokens.doc import Doc +from spacy.tokens.span import Span + +from ._dependency_patterns import dependency_matcher_patterns +from ._parameter_dependencies import ( Action, APIDependencies, Condition, Dependency, - Parameter, ParameterHasValue, ParameterIsIgnored, ParameterIsIllegal, ParameterIsNone, ) -from spacy.matcher import DependencyMatcher -from spacy.tokens import Token -from spacy.tokens.doc import Doc -from spacy.tokens.span import Span - -from ._dependency_patterns import dependency_matcher_patterns from ._preprocess_docstring import preprocess_docstring PIPELINE = "en_core_web_sm" @@ -196,7 +195,7 @@ def get_dependencies(api: API) -> APIDependencies: parameters = function.parameters all_dependencies[function_name] = {} for parameter in parameters: - docstring = parameter.docstring.description + docstring = parameter.documentation.description docstring_preprocessed = preprocess_docstring(docstring) doc = nlp(docstring_preprocessed) param_dependencies = [] diff --git a/package-parser/package_parser/model/api/_parameter_dependencies.py b/package-parser/package_parser/processing/dependencies/_parameter_dependencies.py similarity index 97% rename from package-parser/package_parser/model/api/_parameter_dependencies.py rename to package-parser/package_parser/processing/dependencies/_parameter_dependencies.py index 417cd7b44..55570f338 100644 --- a/package-parser/package_parser/model/api/_parameter_dependencies.py +++ b/package-parser/package_parser/processing/dependencies/_parameter_dependencies.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Any, Dict -from package_parser.model.api import Parameter +from package_parser.processing.api.model import Parameter @dataclass diff --git a/package-parser/package_parser/processing/usages/_ast_visitor.py b/package-parser/package_parser/processing/usages/_ast_visitor.py index 3df798290..d6961500e 100644 --- a/package-parser/package_parser/processing/usages/_ast_visitor.py +++ b/package-parser/package_parser/processing/usages/_ast_visitor.py @@ -4,7 +4,7 @@ import astroid from astroid.arguments import CallSite from astroid.helpers import safe_infer -from package_parser.model.usages import UsageCountStore +from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import parent_id diff --git a/package-parser/package_parser/processing/usages/_find_usages.py b/package-parser/package_parser/processing/usages/_find_usages.py index 7ad946565..99184c71e 100644 --- a/package-parser/package_parser/processing/usages/_find_usages.py +++ b/package-parser/package_parser/processing/usages/_find_usages.py @@ -6,9 +6,9 @@ import astroid from astroid.builder import AstroidBuilder +from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import ASTWalker, list_files, parse_python_code -from ...model.usages import UsageCountStore from ._ast_visitor import _UsageFinder diff --git a/package-parser/package_parser/model/usages/__init__.py b/package-parser/package_parser/processing/usages/model/__init__.py similarity index 100% rename from package-parser/package_parser/model/usages/__init__.py rename to package-parser/package_parser/processing/usages/model/__init__.py diff --git a/package-parser/package_parser/model/usages/_usages.py b/package-parser/package_parser/processing/usages/model/_usages.py similarity index 100% rename from package-parser/package_parser/model/usages/_usages.py rename to package-parser/package_parser/processing/usages/model/_usages.py diff --git a/package-parser/pyproject.toml b/package-parser/pyproject.toml index ab7343473..d378fa9ba 100644 --- a/package-parser/pyproject.toml +++ b/package-parser/pyproject.toml @@ -23,6 +23,8 @@ pytest = "^7.1.2" pytest-cov = "^3.0.0" [tool.mypy] +python_version = "3.10" +no_site_packages = true ignore_missing_imports = true disallow_untyped-calls = true disallow_untyped-defs = true diff --git a/package-parser/tests/__init__.py b/package-parser/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/__init__.py b/package-parser/tests/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/generate_annotations/__init__.py b/package-parser/tests/commands/generate_annotations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/get_api/__init__.py b/package-parser/tests/commands/get_api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/get_dependencies/__init__.py b/package-parser/tests/commands/get_dependencies/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/model/__init__.py b/package-parser/tests/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/model/test_annotations.py b/package-parser/tests/processing/annotations/model/test_annotations.py similarity index 99% rename from package-parser/tests/model/test_annotations.py rename to package-parser/tests/processing/annotations/model/test_annotations.py index b7011bd78..b217eeb84 100644 --- a/package-parser/tests/model/test_annotations.py +++ b/package-parser/tests/processing/annotations/model/test_annotations.py @@ -1,4 +1,4 @@ -from package_parser.model.annotations import ( +from package_parser.processing.annotations.model import ( ANNOTATION_SCHEMA_VERSION, AbstractAnnotation, AnnotationStore, diff --git a/package-parser/tests/commands/generate_annotations/test_generate_annotations.py b/package-parser/tests/processing/annotations/test_generate_annotations.py similarity index 91% rename from package-parser/tests/commands/generate_annotations/test_generate_annotations.py rename to package-parser/tests/processing/annotations/test_generate_annotations.py index 6cbf01832..065031510 100644 --- a/package-parser/tests/commands/generate_annotations/test_generate_annotations.py +++ b/package-parser/tests/processing/annotations/test_generate_annotations.py @@ -2,9 +2,9 @@ import os import pytest -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore from package_parser.processing.annotations import generate_annotations +from package_parser.processing.api.model import API +from package_parser.processing.usages.model import UsageCountStore @pytest.mark.parametrize( diff --git a/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py b/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py new file mode 100644 index 000000000..a23218b9c --- /dev/null +++ b/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py @@ -0,0 +1,54 @@ +import pytest +from package_parser.processing.api.documentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, +) + + +@pytest.mark.parametrize( + "class_documentation", + [ + ClassDocumentation(), + ClassDocumentation(description="foo", full_docstring="foo bar"), + ], +) +def test_dict_conversion_for_class_documentation( + class_documentation: ClassDocumentation, +): + assert ( + ClassDocumentation.from_dict(class_documentation.to_dict()) + == class_documentation + ) + + +@pytest.mark.parametrize( + "function_documentation", + [ + FunctionDocumentation(), + FunctionDocumentation(description="foo", full_docstring="foo bar"), + ], +) +def test_dict_conversion_for_function_documentation( + function_documentation: FunctionDocumentation, +): + assert ( + FunctionDocumentation.from_dict(function_documentation.to_dict()) + == function_documentation + ) + + +@pytest.mark.parametrize( + "parameter_documentation", + [ + ParameterDocumentation(), + ParameterDocumentation(type="int", default_value="1", description="foo bar"), + ], +) +def test_dict_conversion_for_parameter_documentation( + parameter_documentation: ParameterDocumentation, +): + assert ( + ParameterDocumentation.from_dict(parameter_documentation.to_dict()) + == parameter_documentation + ) diff --git a/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py new file mode 100644 index 000000000..be6cf75cf --- /dev/null +++ b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py @@ -0,0 +1,305 @@ +import astroid +import pytest +from package_parser.processing.api.documentation import ( + ClassDocumentation, + FunctionDocumentation, + NumpyDocParser, + ParameterDocumentation, +) + + +@pytest.fixture +def numpydoc_parser() -> NumpyDocParser: + return NumpyDocParser() + + +# language=python +class_with_documentation = ''' +class C: + """ + Lorem ipsum. + + Dolor sit amet. + """ + + def __init__(self): + pass +''' + +# language=python +class_without_documentation = """ +class C: + pass +""" + + +@pytest.mark.parametrize( + "python_code, expected_class_documentation", + [ + ( + class_with_documentation, + ClassDocumentation( + description="Lorem ipsum.\n\nDolor sit amet.", + full_docstring="Lorem ipsum.\n\nDolor sit amet.", + ), + ), + ( + class_without_documentation, + ClassDocumentation(description="", full_docstring=""), + ), + ], + ids=[ + "class with documentation", + "class without documentation", + ], +) +def test_get_class_documentation( + numpydoc_parser: NumpyDocParser, + python_code: str, + expected_class_documentation: ClassDocumentation, +): + node = astroid.extract_node(python_code) + + assert isinstance(node, astroid.ClassDef) + assert numpydoc_parser.get_class_documentation(node) == expected_class_documentation + + +# language=python +function_with_documentation = ''' +def f(): + """ + Lorem ipsum. + + Dolor sit amet. + """ + + pass +''' + +# language=python +function_without_documentation = """ +def f(): + pass +""" + + +@pytest.mark.parametrize( + "python_code, expected_function_documentation", + [ + ( + function_with_documentation, + FunctionDocumentation( + description="Lorem ipsum.\n\nDolor sit amet.", + full_docstring="Lorem ipsum.\n\nDolor sit amet.", + ), + ), + ( + function_without_documentation, + FunctionDocumentation(description="", full_docstring=""), + ), + ], + ids=[ + "function with documentation", + "function without documentation", + ], +) +def test_get_function_documentation( + numpydoc_parser: NumpyDocParser, + python_code: str, + expected_function_documentation: FunctionDocumentation, +): + node = astroid.extract_node(python_code) + + assert isinstance(node, astroid.FunctionDef) + assert ( + numpydoc_parser.get_function_documentation(node) + == expected_function_documentation + ) + + +# language=python +class_with_parameters = ''' +class C: + """ + Lorem ipsum. + + Dolor sit amet. + + Parameters + ---------- + p : int, default=1 + foo + """ + + def __init__(self, p: int = 1): + pass +''' + +# language=python +function_with_parameters = ''' +def f( + no_type_no_default, + type_no_default, + optional_unknown_default: int = 0, + with_default_syntax_1: int = 1, + with_default_syntax_2: int = 2, + with_default_syntax_3: int = 3, + grouped_parameter_1: int = 4, + grouped_parameter_2: int = 4 +): + """ + Lorem ipsum. + + Dolor sit amet. + + Parameters + ---------- + no_type_no_default + foo: no_type_no_default + type_no_default : int + foo: type_no_default + optional_unknown_default : int, optional + foo: optional_unknown_default + with_default_syntax_1 : int, default 1 + foo: with_default_syntax_1 + with_default_syntax_2 : int, default: 2 + foo: with_default_syntax_2 + with_default_syntax_3 : int, default=3 + foo: with_default_syntax_3 + grouped_parameter_1, grouped_parameter_2 : int, default=4 + foo: grouped_parameter_1 and grouped_parameter_2 + """ + + pass +''' + + +@pytest.mark.parametrize( + "python_code, parameter_name, expected_parameter_documentation", + [ + ( + class_with_parameters, + "p", + ParameterDocumentation( + type="int", + default_value="1", + description="foo", + ), + ), + ( + class_with_parameters, + "missing", + ParameterDocumentation( + type="", + default_value="", + description="", + ), + ), + ( + function_with_parameters, + "no_type_no_default", + ParameterDocumentation( + type="", + default_value="", + description="foo: no_type_no_default", + ), + ), + ( + function_with_parameters, + "type_no_default", + ParameterDocumentation( + type="int", + default_value="", + description="foo: type_no_default", + ), + ), + ( + function_with_parameters, + "optional_unknown_default", + ParameterDocumentation( + type="int", + default_value="", + description="foo: optional_unknown_default", + ), + ), + ( + function_with_parameters, + "with_default_syntax_1", + ParameterDocumentation( + type="int", + default_value="1", + description="foo: with_default_syntax_1", + ), + ), + ( + function_with_parameters, + "with_default_syntax_2", + ParameterDocumentation( + type="int", default_value="2", description="foo: with_default_syntax_2" + ), + ), + ( + function_with_parameters, + "with_default_syntax_3", + ParameterDocumentation( + type="int", default_value="3", description="foo: with_default_syntax_3" + ), + ), + ( + function_with_parameters, + "grouped_parameter_1", + ParameterDocumentation( + type="int", + default_value="4", + description="foo: grouped_parameter_1 and grouped_parameter_2", + ), + ), + ( + function_with_parameters, + "grouped_parameter_2", + ParameterDocumentation( + type="int", + default_value="4", + description="foo: grouped_parameter_1 and grouped_parameter_2", + ), + ), + ( + function_with_parameters, + "missing", + ParameterDocumentation(type="", default_value="", description=""), + ), + ], + ids=[ + "existing class parameter", + "missing class parameter", + "function parameter with no type and no default", + "function parameter with type and no default", + "function parameter with optional unknown default", + "function parameter with default syntax 1 (just space)", + "function parameter with default syntax 2 (colon)", + "function parameter with default syntax 3 (equals)", + "function parameter with grouped parameters 1", + "function parameter with grouped parameters 2", + "missing function parameter", + ], +) +def test_get_parameter_documentation( + numpydoc_parser: NumpyDocParser, + python_code: str, + parameter_name: str, + expected_parameter_documentation: ParameterDocumentation, +): + node = astroid.extract_node(python_code) + assert isinstance(node, astroid.ClassDef) or isinstance(node, astroid.FunctionDef) + + # Find the constructor + if isinstance(node, astroid.ClassDef): + for method in node.mymethods(): + if method.name == "__init__": + node = method + + assert isinstance(node, astroid.FunctionDef) + assert ( + numpydoc_parser.get_parameter_documentation(node, parameter_name) + == expected_parameter_documentation + ) diff --git a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py new file mode 100644 index 000000000..a1fd66d2a --- /dev/null +++ b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py @@ -0,0 +1,81 @@ +import astroid +import pytest +from package_parser.processing.api.documentation import get_full_docstring + +# language=python +class_with_multi_line_documentation = ''' +class C: + """ + Lorem ipsum. + + Dolor sit amet. + """ + + pass +''' + +# language=python +class_with_single_line_documentation = ''' +class C: + """Lorem ipsum.""" + + pass +''' + +# language=python +class_without_documentation = """ +class C: + pass +""" + +# language=python +function_with_multi_line_documentation = ''' +def f(): + """ + Lorem ipsum. + + Dolor sit amet. + """ + + pass +''' + +# language=python +function_with_single_line_documentation = ''' +def f(): + """Lorem ipsum.""" + + pass +''' + +# language=python +function_without_documentation = """ +def f(): + pass +""" + + +@pytest.mark.parametrize( + "python_code, expected_docstring", + [ + (class_with_multi_line_documentation, "Lorem ipsum.\n\nDolor sit amet."), + (class_with_single_line_documentation, "Lorem ipsum."), + (class_without_documentation, ""), + (function_with_multi_line_documentation, "Lorem ipsum.\n\nDolor sit amet."), + (function_with_single_line_documentation, "Lorem ipsum."), + (function_without_documentation, ""), + ], + ids=[ + "class with multi line documentation", + "class with single line documentation", + "class without documentation", + "function with multi line documentation", + "function with single line documentation", + "function without documentation", + ], +) +def test_get_full_docstring(python_code: str, expected_docstring: str): + node = astroid.extract_node(python_code) + + assert isinstance(node, astroid.ClassDef) or isinstance(node, astroid.FunctionDef) + assert get_full_docstring(node) == expected_docstring diff --git a/package-parser/tests/commands/get_api/test_boundaries.py b/package-parser/tests/processing/api/test_boundaries.py similarity index 97% rename from package-parser/tests/commands/get_api/test_boundaries.py rename to package-parser/tests/processing/api/test_boundaries.py index a6f5439b8..7001faad3 100644 --- a/package-parser/tests/commands/get_api/test_boundaries.py +++ b/package-parser/tests/processing/api/test_boundaries.py @@ -1,5 +1,5 @@ import pytest -from package_parser.model.api._types import BoundaryType +from package_parser.processing.api.model import BoundaryType @pytest.mark.parametrize( diff --git a/package-parser/tests/commands/get_api/test_enums.py b/package-parser/tests/processing/api/test_enums.py similarity index 96% rename from package-parser/tests/commands/get_api/test_enums.py rename to package-parser/tests/processing/api/test_enums.py index 1b9d9c8e1..3a37b06a1 100644 --- a/package-parser/tests/commands/get_api/test_enums.py +++ b/package-parser/tests/processing/api/test_enums.py @@ -1,7 +1,7 @@ from typing import Optional import pytest -from package_parser.model.api._types import EnumType +from package_parser.processing.api.model import EnumType @pytest.mark.parametrize( diff --git a/package-parser/tests/commands/get_api/test_file_filters.py b/package-parser/tests/processing/api/test_file_filters.py similarity index 100% rename from package-parser/tests/commands/get_api/test_file_filters.py rename to package-parser/tests/processing/api/test_file_filters.py diff --git a/package-parser/tests/commands/get_api/test_types.py b/package-parser/tests/processing/api/test_types.py similarity index 91% rename from package-parser/tests/commands/get_api/test_types.py rename to package-parser/tests/processing/api/test_types.py index d87589c29..da10649dd 100644 --- a/package-parser/tests/commands/get_api/test_types.py +++ b/package-parser/tests/processing/api/test_types.py @@ -1,7 +1,8 @@ from typing import Any import pytest -from package_parser.model.api import ParameterAndResultDocstring, Type +from package_parser.processing.api.documentation import ParameterDocumentation +from package_parser.processing.api.model import Type @pytest.mark.parametrize( @@ -70,7 +71,7 @@ ], ) def test_union_from_string(docstring_type: str, expected: dict[str, Any]): - result = Type(ParameterAndResultDocstring(docstring_type, "")) + result = Type(ParameterDocumentation(docstring_type, "", "")) assert result.to_json() == expected @@ -103,7 +104,7 @@ def test_union_from_string(docstring_type: str, expected: dict[str, Any]): ], ) def test_boundary_from_string(docstring_type: str, expected: dict[str, Any]): - assert Type(ParameterAndResultDocstring("", docstring_type)).to_json() == expected + assert Type(ParameterDocumentation("", "", docstring_type)).to_json() == expected @pytest.mark.parametrize( @@ -136,7 +137,9 @@ def test_boundary_and_union_from_string( ): assert ( Type( - ParameterAndResultDocstring(docstring_type, docstring_description) + ParameterDocumentation( + type=docstring_type, default_value="", description=docstring_description + ) ).to_json() == expected ) diff --git a/package-parser/tests/commands/get_dependencies/test_get_dependency.py b/package-parser/tests/processing/dependencies/test_get_dependency.py similarity index 90% rename from package-parser/tests/commands/get_dependencies/test_get_dependency.py rename to package-parser/tests/processing/dependencies/test_get_dependency.py index e4bd29835..8db90eb50 100644 --- a/package-parser/tests/commands/get_dependencies/test_get_dependency.py +++ b/package-parser/tests/processing/dependencies/test_get_dependency.py @@ -1,18 +1,15 @@ import spacy -from package_parser.model.api import ( +from package_parser.processing.api.documentation import ParameterDocumentation +from package_parser.processing.api.model import Parameter, ParameterAssignment +from package_parser.processing.dependencies import ( Action, Condition, Dependency, - Parameter, - ParameterAndResultDocstring, - ParameterAssignment, + DependencyExtractor, ParameterHasValue, ParameterIsIgnored, ParameterIsIllegal, ParameterIsNone, -) -from package_parser.processing.dependencies import ( - DependencyExtractor, extract_action, extract_condition, extract_lefts_and_rights, @@ -103,8 +100,10 @@ def test_extract_dependencies_from_docstring_pattern_adverbial_clause(): default_value=None, assigned_by=ParameterAssignment.NAME_ONLY, is_public=True, - docstring=ParameterAndResultDocstring( - type="param possible types", description=param_docstring_nlp.text + documentation=ParameterDocumentation( + type="param possible types", + default_value="", + description=param_docstring_nlp.text, ), ) dependee_param = Parameter( @@ -114,8 +113,10 @@ def test_extract_dependencies_from_docstring_pattern_adverbial_clause(): default_value=None, assigned_by=ParameterAssignment.NAME_ONLY, is_public=True, - docstring=ParameterAndResultDocstring( - type="param possible types", description="param probability docstring" + documentation=ParameterDocumentation( + type="param possible types", + default_value="", + description="param probability docstring", ), ) func_params = [dependent_param, dependee_param] diff --git a/package-parser/tests/commands/get_dependencies/test_preprocess_docstring.py b/package-parser/tests/processing/dependencies/test_preprocess_docstring.py similarity index 100% rename from package-parser/tests/commands/get_dependencies/test_preprocess_docstring.py rename to package-parser/tests/processing/dependencies/test_preprocess_docstring.py diff --git a/package-parser/tests/model/test_usages.py b/package-parser/tests/processing/usages/model/test_usages.py similarity index 99% rename from package-parser/tests/model/test_usages.py rename to package-parser/tests/processing/usages/model/test_usages.py index be4079139..703a01d49 100644 --- a/package-parser/tests/model/test_usages.py +++ b/package-parser/tests/processing/usages/model/test_usages.py @@ -1,7 +1,10 @@ from typing import Any import pytest -from package_parser.model.usages import USAGES_SCHEMA_VERSION, UsageCountStore +from package_parser.processing.usages.model import ( + USAGES_SCHEMA_VERSION, + UsageCountStore, +) @pytest.fixture