From 1de1a34552910b89e23c0fbea6dca34c2880a331 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 09:44:09 +0200 Subject: [PATCH 01/16] refactor(parser): move function that creates parameter list --- .../processing/api/_ast_visitor.py | 109 +----------------- .../processing/api/_get_parameter_list.py | 109 ++++++++++++++++++ 2 files changed, 113 insertions(+), 105 deletions(-) create mode 100644 package-parser/package_parser/processing/api/_get_parameter_list.py diff --git a/package-parser/package_parser/processing/api/_ast_visitor.py b/package-parser/package_parser/processing/api/_ast_visitor.py index 856fa49a6..3396fa6bc 100644 --- a/package-parser/package_parser/processing/api/_ast_visitor.py +++ b/package-parser/package_parser/processing/api/_ast_visitor.py @@ -7,6 +7,7 @@ from astroid.context import InferenceContext from astroid.helpers import safe_infer from numpydoc.docscrape import NumpyDocString + from package_parser.model.api import ( API, Class, @@ -14,13 +15,10 @@ Function, Import, Module, - Parameter, - ParameterAndResultDocstring, - ParameterAssignment, ) from package_parser.utils import parent_qualified_name - from ._file_filters import _is_init_file +from ._get_parameter_list import _get_parameter_list class _AstVisitor: @@ -184,8 +182,8 @@ def enter_functiondef(self, function_node: astroid.FunctionDef) -> None: self.__get_function_id(function_node.name, decorator_names), qname, decorator_names, - self.__function_parameters( - function_node, is_public, qname, self.__get_id(function_node.name) + _get_parameter_list( + function_node, self.__get_id(function_node.name), qname, is_public, ), [], # TODO: results is_public, @@ -227,105 +225,6 @@ def __description(numpydoc: NumpyDocString) -> str: result += "\n".join(numpydoc["Extended Summary"]) return result - @staticmethod - def __function_parameters( - node: astroid.FunctionDef, - function_is_public: bool, - function_qname: str, - function_id: str, - ) -> list[Parameter]: - parameters = node.args - n_implicit_parameters = node.implicit_parameters() - - # For constructors (__init__ functions) the parameters are described on the class - if node.name == "__init__" and isinstance(node.parent, astroid.ClassDef): - docstring = node.parent.doc - else: - docstring = node.doc - function_numpydoc = NumpyDocString(inspect.cleandoc(docstring or "")) - - # Arguments that can be specified positionally only ( f(1) works but not f(x=1) ) - result = [ - Parameter( - id_=function_id + "/" + it.name, - name=it.name, - qname=function_qname + "." + it.name, - default_value=None, - assigned_by=ParameterAssignment.POSITION_ONLY, - is_public=function_is_public, - docstring=_AstVisitor.__parameter_docstring(function_numpydoc, it.name), - ) - for it in parameters.posonlyargs - ] - - # Arguments that can be specified positionally or by name ( f(1) and f(x=1) both work ) - result += [ - Parameter( - function_id + "/" + it.name, - it.name, - function_qname + "." + it.name, - _AstVisitor.__parameter_default( - parameters.defaults, - index - len(parameters.args) + len(parameters.defaults), - ), - ParameterAssignment.POSITION_OR_NAME, - function_is_public, - _AstVisitor.__parameter_docstring(function_numpydoc, it.name), - ) - for index, it in enumerate(parameters.args) - ] - - # Arguments that can be specified by name only ( f(x=1) works but not f(1) ) - result += [ - Parameter( - function_id + "/" + it.name, - it.name, - function_qname + "." + it.name, - _AstVisitor.__parameter_default( - parameters.kw_defaults, - index - len(parameters.kwonlyargs) + len(parameters.kw_defaults), - ), - ParameterAssignment.NAME_ONLY, - function_is_public, - _AstVisitor.__parameter_docstring(function_numpydoc, it.name), - ) - for index, it in enumerate(parameters.kwonlyargs) - ] - - implicit_parameters = result[:n_implicit_parameters] - for implicit_parameter in implicit_parameters: - implicit_parameter.assigned_by = ParameterAssignment.IMPLICIT - - return result - - @staticmethod - def __parameter_default( - defaults: list[astroid.NodeNG], default_index: int - ) -> Optional[str]: - if 0 <= default_index < len(defaults): - default = defaults[default_index] - if default is None: - return None - return default.as_string() - else: - return None - - @staticmethod - def __parameter_docstring( - function_numpydoc: NumpyDocString, parameter_name: str - ) -> ParameterAndResultDocstring: - parameters_numpydoc = function_numpydoc["Parameters"] - candidate_parameters_numpydoc = [ - it for it in parameters_numpydoc if it.name == parameter_name - ] - - if len(candidate_parameters_numpydoc) > 0: - last_parameter_numpydoc = candidate_parameters_numpydoc[-1] - return ParameterAndResultDocstring( - last_parameter_numpydoc.type, "\n".join(last_parameter_numpydoc.desc) - ) - - return ParameterAndResultDocstring("", "") def is_public(self, name: str, qualified_name: str) -> bool: if name.startswith("_") and not name.endswith("__"): diff --git a/package-parser/package_parser/processing/api/_get_parameter_list.py b/package-parser/package_parser/processing/api/_get_parameter_list.py new file mode 100644 index 000000000..52ca0ce4a --- /dev/null +++ b/package-parser/package_parser/processing/api/_get_parameter_list.py @@ -0,0 +1,109 @@ +import inspect +from typing import Optional + +import astroid +from numpydoc.docscrape import NumpyDocString + +from package_parser.model.api import Parameter, ParameterAssignment, ParameterAndResultDocstring + + +def _get_parameter_list( + node: astroid.FunctionDef, + function_id: str, + function_qname: str, + function_is_public: bool, +) -> list[Parameter]: + parameters = node.args + n_implicit_parameters = node.implicit_parameters() + + # For constructors (__init__ functions) the parameters are described on the class + if node.name == "__init__" and isinstance(node.parent, astroid.ClassDef): + docstring = node.parent.doc + else: + docstring = node.doc + function_numpydoc = NumpyDocString(inspect.cleandoc(docstring or "")) + + # Arguments that can be specified positionally only ( f(1) works but not f(x=1) ) + result = [ + Parameter( + id_=function_id + "/" + it.name, + name=it.name, + qname=function_qname + "." + it.name, + default_value=None, + assigned_by=ParameterAssignment.POSITION_ONLY, + is_public=function_is_public, + docstring=_get_parameter_docstring(function_numpydoc, it.name), + ) + for it in parameters.posonlyargs + ] + + # Arguments that can be specified positionally or by name ( f(1) and f(x=1) both work ) + result += [ + Parameter( + function_id + "/" + it.name, + it.name, + function_qname + "." + it.name, + _get_parameter_default( + parameters.defaults, + index - len(parameters.args) + len(parameters.defaults), + ), + ParameterAssignment.POSITION_OR_NAME, + function_is_public, + _get_parameter_docstring(function_numpydoc, it.name), + ) + for index, it in enumerate(parameters.args) + ] + + # Arguments that can be specified by name only ( f(x=1) works but not f(1) ) + result += [ + Parameter( + function_id + "/" + it.name, + it.name, + function_qname + "." + it.name, + _get_parameter_default( + parameters.kw_defaults, + index - len(parameters.kwonlyargs) + len(parameters.kw_defaults), + ), + ParameterAssignment.NAME_ONLY, + function_is_public, + _get_parameter_docstring(function_numpydoc, it.name), + ) + for index, it in enumerate(parameters.kwonlyargs) + ] + + implicit_parameters = result[:n_implicit_parameters] + for implicit_parameter in implicit_parameters: + implicit_parameter.assigned_by = ParameterAssignment.IMPLICIT + + return result + + +def _get_parameter_default( + defaults: list[astroid.NodeNG], + default_index: int +) -> Optional[str]: + if 0 <= default_index < len(defaults): + default = defaults[default_index] + if default is None: + return None + return default.as_string() + else: + return None + + +def _get_parameter_docstring( + function_numpydoc: NumpyDocString, + parameter_name: str +) -> ParameterAndResultDocstring: + parameters_numpydoc = function_numpydoc["Parameters"] + candidate_parameters_numpydoc = [ + it for it in parameters_numpydoc if it.name == parameter_name + ] + + if len(candidate_parameters_numpydoc) > 0: + last_parameter_numpydoc = candidate_parameters_numpydoc[-1] + return ParameterAndResultDocstring( + last_parameter_numpydoc.type, "\n".join(last_parameter_numpydoc.desc) + ) + + return ParameterAndResultDocstring("", "") From bdf1cb8a94b573a0c560e1adbf3883fa63cddfc8 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 10:31:06 +0200 Subject: [PATCH 02/16] refactor(parser): documentation parsing strategy --- package-parser/.gitignore | 3 ++ .../_AbstractDocumentationParsingStrategy.py | 47 +++++++++++++++++++ .../api/documentation/_NumpydocParser.py | 16 +++++++ .../processing/api/documentation/__init__.py | 1 + package-parser/tests/__init__.py | 0 package-parser/tests/commands/__init__.py | 0 .../commands/generate_annotations/__init__.py | 0 .../tests/commands/get_api/__init__.py | 0 .../commands/get_dependencies/__init__.py | 0 package-parser/tests/model/__init__.py | 0 .../documentation/test_get_full_docstring.py | 45 ++++++++++++++++++ 11 files changed, 112 insertions(+) create mode 100644 package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py create mode 100644 package-parser/package_parser/processing/api/documentation/_NumpydocParser.py create mode 100644 package-parser/package_parser/processing/api/documentation/__init__.py delete mode 100644 package-parser/tests/__init__.py delete mode 100644 package-parser/tests/commands/__init__.py delete mode 100644 package-parser/tests/commands/generate_annotations/__init__.py delete mode 100644 package-parser/tests/commands/get_api/__init__.py delete mode 100644 package-parser/tests/commands/get_dependencies/__init__.py delete mode 100644 package-parser/tests/model/__init__.py create mode 100644 package-parser/tests/processing/api/documentation/test_get_full_docstring.py diff --git a/package-parser/.gitignore b/package-parser/.gitignore index 761e860a8..f304e6a07 100644 --- a/package-parser/.gitignore +++ b/package-parser/.gitignore @@ -25,6 +25,9 @@ dist/ htmlcov/ .coverage +# Test __init__.py files +tests/**/__init__.py + # Output of this tool out/ tmp/ diff --git a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py new file mode 100644 index 000000000..cc1ed9064 --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass + +import astroid + + +class AbstractDocumentationParsingStrategy(ABC): + + @abstractmethod + def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: + pass + + @abstractmethod + def get_function_documentation(self, function_node: astroid.FunctionDef) -> FunctionDocumentation: + pass + + @abstractmethod + def get_parameter_documentation(self, function_node: astroid.FunctionDef, + parameter_name: str) -> ParameterDocumentation: + pass + + +@dataclass +class ClassDocumentation: + description: str + full_docstring: str + + +@dataclass +class FunctionDocumentation: + description: str + full_docstring: str + + +@dataclass +class ParameterDocumentation: + type: str + description: str + + +def get_full_docstring(declaration: astroid.ClassDef | astroid.FunctionDef) -> str: + doc_node = declaration.doc_node + if doc_node is None: + return "" + return doc_node.value diff --git a/package-parser/package_parser/processing/api/documentation/_NumpydocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpydocParser.py new file mode 100644 index 000000000..babf5ec60 --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_NumpydocParser.py @@ -0,0 +1,16 @@ +import astroid + +from package_parser.processing.api.documentation._AbstractDocumentationParsingStrategy import \ + AbstractDocumentationParsingStrategy, ParameterDocumentation, FunctionDocumentation, ClassDocumentation + + +class NumpydocParser(AbstractDocumentationParsingStrategy): + def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: + pass + + def get_function_documentation(self, function_node: astroid.FunctionDef) -> FunctionDocumentation: + pass + + def get_parameter_documentation(self, function_node: astroid.FunctionDef, + parameter_name: str) -> ParameterDocumentation: + pass diff --git a/package-parser/package_parser/processing/api/documentation/__init__.py b/package-parser/package_parser/processing/api/documentation/__init__.py new file mode 100644 index 000000000..37ce7df4e --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/__init__.py @@ -0,0 +1 @@ +from ._NumpydocParser import NumpydocParser diff --git a/package-parser/tests/__init__.py b/package-parser/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/__init__.py b/package-parser/tests/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/generate_annotations/__init__.py b/package-parser/tests/commands/generate_annotations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/get_api/__init__.py b/package-parser/tests/commands/get_api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/commands/get_dependencies/__init__.py b/package-parser/tests/commands/get_dependencies/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/model/__init__.py b/package-parser/tests/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py new file mode 100644 index 000000000..2245a55d0 --- /dev/null +++ b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py @@ -0,0 +1,45 @@ +import astroid +import pytest + +# language=python +from package_parser.processing.api.documentation._AbstractDocumentationParsingStrategy import get_full_docstring + +test_class_with_documentation = ''' +class C: + """Lorem ipsum.""" + pass +''' + +# language=python +test_function_with_documentation = ''' +def f(): + """Lorem ipsum.""" + pass +''' + +test_class_without_documentation = ''' +class C: + pass +''' + +# language=python +test_function_without_documentation = ''' +def f(): + pass +''' + + +@pytest.mark.parametrize( + "python_code, expected_docstring", + [ + (test_class_with_documentation, "Lorem ipsum."), + (test_function_with_documentation, "Lorem ipsum."), + (test_class_without_documentation, ""), + (test_function_without_documentation, ""), + ] +) +def test_get_full_docstring(python_code: str, expected_docstring: str): + node = astroid.extract_node(python_code) + + assert isinstance(node, astroid.ClassDef) or isinstance(node, astroid.FunctionDef) + assert get_full_docstring(node) == expected_docstring From 439e2db8a81c256a72be4f3a1e29cfacc37d6916 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 12:49:51 +0200 Subject: [PATCH 03/16] refactor(parser): extract parsing of NumpyDoc into strategy --- .../_AbstractDocumentationParsingStrategy.py | 15 +- .../api/documentation/_NumpyDocParser.py | 102 ++++++++ .../api/documentation/_NumpydocParser.py | 16 -- .../processing/api/documentation/__init__.py | 4 +- .../api/documentation/_get_full_docstring.py | 15 ++ .../api/documentation/test_NumpyDocParser.py | 234 ++++++++++++++++++ .../documentation/test_get_full_docstring.py | 51 +++- 7 files changed, 400 insertions(+), 37 deletions(-) create mode 100644 package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py delete mode 100644 package-parser/package_parser/processing/api/documentation/_NumpydocParser.py create mode 100644 package-parser/package_parser/processing/api/documentation/_get_full_docstring.py create mode 100644 package-parser/tests/processing/api/documentation/test_NumpyDocParser.py diff --git a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py index cc1ed9064..be291bed4 100644 --- a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py +++ b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py @@ -17,8 +17,11 @@ def get_function_documentation(self, function_node: astroid.FunctionDef) -> Func pass @abstractmethod - def get_parameter_documentation(self, function_node: astroid.FunctionDef, - parameter_name: str) -> ParameterDocumentation: + def get_parameter_documentation( + self, + function_node: astroid.FunctionDef, + parameter_name: str + ) -> ParameterDocumentation: pass @@ -37,11 +40,5 @@ class FunctionDocumentation: @dataclass class ParameterDocumentation: type: str + default_value: str description: str - - -def get_full_docstring(declaration: astroid.ClassDef | astroid.FunctionDef) -> str: - doc_node = declaration.doc_node - if doc_node is None: - return "" - return doc_node.value diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py new file mode 100644 index 000000000..561304a37 --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -0,0 +1,102 @@ +import re + +import astroid +import numpydoc.docscrape +from numpydoc.docscrape import NumpyDocString + +from ._AbstractDocumentationParsingStrategy import AbstractDocumentationParsingStrategy, ParameterDocumentation, \ + FunctionDocumentation, ClassDocumentation +from ._get_full_docstring import get_full_docstring + + +class NumpyDocParser(AbstractDocumentationParsingStrategy): + def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: + docstring = get_full_docstring(class_node) + + return ClassDocumentation( + description=_get_description(docstring), + full_docstring=docstring + ) + + def get_function_documentation(self, function_node: astroid.FunctionDef) -> FunctionDocumentation: + docstring = get_full_docstring(function_node) + + return FunctionDocumentation( + description=_get_description(docstring), + full_docstring=docstring + ) + + def get_parameter_documentation( + self, + function_node: astroid.FunctionDef, + parameter_name: str + ) -> ParameterDocumentation: + + # For constructors (__init__ functions) the parameters are described on the class + if function_node.name == "__init__" and isinstance(function_node.parent, astroid.ClassDef): + docstring = get_full_docstring(function_node.parent) + else: + docstring = get_full_docstring(function_node) + + # Find matching parameter docstrings. Numpydoc allows multiple parameters to be documented at once. See + # https://numpydoc.readthedocs.io/en/latest/format.html#parameters for more information. + function_numpydoc = NumpyDocString(docstring) + all_parameters_numpydoc: list[numpydoc.docscrape.Parameter] = function_numpydoc.get("Parameters", []) + matching_parameters_numpydoc = [ + it for it in all_parameters_numpydoc + if _is_matching_parameter_numpydoc(it, parameter_name) + ] + + if len(matching_parameters_numpydoc) == 0: + return ParameterDocumentation( + type="", + default_value="", + description="" + ) + + last_parameter_numpydoc = matching_parameters_numpydoc[-1] + type, default_value = _get_type_and_default_value(last_parameter_numpydoc) + return ParameterDocumentation( + type=type, + default_value=default_value, + description="\n".join([line.strip() for line in last_parameter_numpydoc.desc]) + ) + + +def _get_description(docstring: str) -> str: + """ + Returns the concatenated summary and extended summary parts of the given docstring or an empty string if these parts + are blank. + """ + + numpydoc_ = NumpyDocString(docstring) + summary: list[str] = numpydoc_.get("Summary", []) + extended_summary: list[str] = numpydoc_.get("Extended Summary", []) + + result = "" + result += "\n".join([line.strip() for line in summary]) + result += "\n\n" + result += "\n".join([line.strip() for line in extended_summary]) + return result.strip() + + +def _is_matching_parameter_numpydoc(parameter_numpydoc: numpydoc.docscrape.Parameter, parameter_name: str) -> bool: + """ + Returns whether the given NumpyDoc applied to the parameter with the given name. + """ + + return any(name.strip() == parameter_name for name in parameter_numpydoc.name.split(",")) + + +def _get_type_and_default_value(parameter_numpydoc: numpydoc.docscrape.Parameter) -> (str, str): + """ + Returns the type and default value for the given NumpyDoc. + """ + + type_ = parameter_numpydoc.type + parts = re.split(r",\s*optional|,\s*default\s*[:=]?", type_) + + if len(parts) != 2: + return type_.strip(), "" + + return parts[0].strip(), parts[1].strip() diff --git a/package-parser/package_parser/processing/api/documentation/_NumpydocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpydocParser.py deleted file mode 100644 index babf5ec60..000000000 --- a/package-parser/package_parser/processing/api/documentation/_NumpydocParser.py +++ /dev/null @@ -1,16 +0,0 @@ -import astroid - -from package_parser.processing.api.documentation._AbstractDocumentationParsingStrategy import \ - AbstractDocumentationParsingStrategy, ParameterDocumentation, FunctionDocumentation, ClassDocumentation - - -class NumpydocParser(AbstractDocumentationParsingStrategy): - def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: - pass - - def get_function_documentation(self, function_node: astroid.FunctionDef) -> FunctionDocumentation: - pass - - def get_parameter_documentation(self, function_node: astroid.FunctionDef, - parameter_name: str) -> ParameterDocumentation: - pass diff --git a/package-parser/package_parser/processing/api/documentation/__init__.py b/package-parser/package_parser/processing/api/documentation/__init__.py index 37ce7df4e..ae818e85e 100644 --- a/package-parser/package_parser/processing/api/documentation/__init__.py +++ b/package-parser/package_parser/processing/api/documentation/__init__.py @@ -1 +1,3 @@ -from ._NumpydocParser import NumpydocParser +from ._AbstractDocumentationParsingStrategy import ClassDocumentation, FunctionDocumentation, ParameterDocumentation +from ._NumpyDocParser import NumpyDocParser +from ._get_full_docstring import get_full_docstring diff --git a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py new file mode 100644 index 000000000..9269c28a9 --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py @@ -0,0 +1,15 @@ +import inspect + +import astroid + + +def get_full_docstring(declaration: astroid.ClassDef | astroid.FunctionDef) -> str: + """ + Returns the full docstring of the given declaration or an empty string if no docstring is available. Indentation is + cleaned up. + """ + + doc_node = declaration.doc_node + if doc_node is None: + return "" + return inspect.cleandoc(doc_node.value) diff --git a/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py new file mode 100644 index 000000000..80e3d7ec5 --- /dev/null +++ b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py @@ -0,0 +1,234 @@ +import astroid +import pytest + +from package_parser.processing.api.documentation import NumpyDocParser, ClassDocumentation, ParameterDocumentation, \ + FunctionDocumentation + + +@pytest.fixture +def numpydoc_parser() -> NumpyDocParser: + return NumpyDocParser() + + +# language=python +class_with_documentation = ''' +class C: + """ + Lorem ipsum. + + Dolor sit amet. + """ + + def __init__(self): + pass +''' + +# language=python +class_without_documentation = ''' +class C: + pass +''' + + +@pytest.mark.parametrize( + "python_code, expected_class_documentation", + [ + + (class_with_documentation, ClassDocumentation( + description="Lorem ipsum.\n\nDolor sit amet.", + full_docstring="Lorem ipsum.\n\nDolor sit amet.")), + (class_without_documentation, ClassDocumentation( + description="", + full_docstring="" + )), + ] +) +def test_get_class_documentation( + numpydoc_parser: NumpyDocParser, + python_code: str, + expected_class_documentation: ClassDocumentation +): + node = astroid.extract_node(python_code) + + assert isinstance(node, astroid.ClassDef) + assert numpydoc_parser.get_class_documentation(node) == expected_class_documentation + + +# language=python +function_with_documentation = ''' +def f(): + """ + Lorem ipsum. + + Dolor sit amet. + """ + + pass +''' + +# language=python +function_without_documentation = ''' +def f(): + pass +''' + + +@pytest.mark.parametrize( + "python_code, expected_function_documentation", + [ + + (function_with_documentation, FunctionDocumentation( + description="Lorem ipsum.\n\nDolor sit amet.", + full_docstring="Lorem ipsum.\n\nDolor sit amet.")), + (function_without_documentation, FunctionDocumentation( + description="", + full_docstring="" + )), + ] +) +def test_get_function_documentation( + numpydoc_parser: NumpyDocParser, + python_code: str, + expected_function_documentation: FunctionDocumentation +): + node = astroid.extract_node(python_code) + + assert isinstance(node, astroid.FunctionDef) + assert numpydoc_parser.get_function_documentation(node) == expected_function_documentation + + +# language=python +class_with_parameters = ''' +class C: + """ + Lorem ipsum. + + Dolor sit amet. + + Parameters + ---------- + p : int, default=1 + foo + """ + + def __init__(self, p: int = 1): + pass +''' + +# language=python +function_with_parameters = ''' +def f( + no_type_no_default, + type_no_default, + optional_unknown_default: int = 0, + with_default_syntax_1: int = 1, + with_default_syntax_2: int = 2, + with_default_syntax_3: int = 3, + grouped_parameter_1: int = 4, + grouped_parameter_2: int = 4 +): + """ + Lorem ipsum. + + Dolor sit amet. + + Parameters + ---------- + no_type_no_default + foo: no_type_no_default + type_no_default : int + foo: type_no_default + optional_unknown_default : int, optional + foo: optional_unknown_default + with_default_syntax_1 : int, default 1 + foo: with_default_syntax_1 + with_default_syntax_2 : int, default: 2 + foo: with_default_syntax_2 + with_default_syntax_3 : int, default=3 + foo: with_default_syntax_3 + grouped_parameter_1, grouped_parameter_2 : int, default=4 + foo: grouped_parameter_1 and grouped_parameter_2 + """ + + pass +''' + + +@pytest.mark.parametrize( + "python_code, parameter_name, expected_parameter_documentation", + [ + + (class_with_parameters, "p", ParameterDocumentation( + type="int", + default_value="1", + description="foo", + )), + (class_with_parameters, "missing", ParameterDocumentation( + type="", + default_value="", + description="", + )), + (function_with_parameters, "no_type_no_default", ParameterDocumentation( + type="", + default_value="", + description="foo: no_type_no_default", + )), + (function_with_parameters, "type_no_default", ParameterDocumentation( + type="int", + default_value="", + description="foo: type_no_default", + )), + (function_with_parameters, "optional_unknown_default", ParameterDocumentation( + type="int", + default_value="", + description="foo: optional_unknown_default", + )), + (function_with_parameters, "with_default_syntax_1", ParameterDocumentation( + type="int", + default_value="1", + description="foo: with_default_syntax_1", + )), + (function_with_parameters, "with_default_syntax_2", ParameterDocumentation( + type="int", + default_value="2", + description="foo: with_default_syntax_2" + )), + (function_with_parameters, "with_default_syntax_3", ParameterDocumentation( + type="int", + default_value="3", + description="foo: with_default_syntax_3" + )), + (function_with_parameters, "grouped_parameter_1", ParameterDocumentation( + type="int", + default_value="4", + description="foo: grouped_parameter_1 and grouped_parameter_2" + )), + (function_with_parameters, "grouped_parameter_2", ParameterDocumentation( + type="int", + default_value="4", + description="foo: grouped_parameter_1 and grouped_parameter_2" + )), + (function_with_parameters, "missing", ParameterDocumentation( + type="", + default_value="", + description="" + )), + ] +) +def test_get_parameter_documentation( + numpydoc_parser: NumpyDocParser, + python_code: str, + parameter_name: str, + expected_parameter_documentation: ParameterDocumentation +): + node = astroid.extract_node(python_code) + assert isinstance(node, astroid.ClassDef) or isinstance(node, astroid.FunctionDef) + + # Find the constructor + if isinstance(node, astroid.ClassDef): + for method in node.mymethods(): + if method.name == "__init__": + node = method + + assert isinstance(node, astroid.FunctionDef) + assert numpydoc_parser.get_parameter_documentation(node, parameter_name) == expected_parameter_documentation diff --git a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py index 2245a55d0..cad48d9c9 100644 --- a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py +++ b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py @@ -1,29 +1,56 @@ import astroid import pytest +from package_parser.processing.api.documentation import get_full_docstring + # language=python -from package_parser.processing.api.documentation._AbstractDocumentationParsingStrategy import get_full_docstring +class_with_multi_line_documentation = ''' +class C: + """ + Lorem ipsum. + + Dolor sit amet. + """ + + pass +''' -test_class_with_documentation = ''' +# language=python +class_with_single_line_documentation = ''' class C: """Lorem ipsum.""" + pass ''' # language=python -test_function_with_documentation = ''' +class_without_documentation = ''' +class C: + pass +''' + +# language=python +function_with_multi_line_documentation = ''' def f(): - """Lorem ipsum.""" + """ + Lorem ipsum. + + Dolor sit amet. + """ + pass ''' -test_class_without_documentation = ''' -class C: +# language=python +function_with_single_line_documentation = ''' +def f(): + """Lorem ipsum.""" + pass ''' # language=python -test_function_without_documentation = ''' +function_without_documentation = ''' def f(): pass ''' @@ -32,10 +59,12 @@ def f(): @pytest.mark.parametrize( "python_code, expected_docstring", [ - (test_class_with_documentation, "Lorem ipsum."), - (test_function_with_documentation, "Lorem ipsum."), - (test_class_without_documentation, ""), - (test_function_without_documentation, ""), + (class_with_multi_line_documentation, "Lorem ipsum.\n\nDolor sit amet."), + (class_with_single_line_documentation, "Lorem ipsum."), + (class_without_documentation, ""), + (function_with_multi_line_documentation, "Lorem ipsum.\n\nDolor sit amet."), + (function_with_single_line_documentation, "Lorem ipsum."), + (function_without_documentation, ""), ] ) def test_get_full_docstring(python_code: str, expected_docstring: str): From f501bf800f3a6a38346832adea1d8a17a4a05367 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 13:00:51 +0200 Subject: [PATCH 04/16] refactor(parser): clean up structure --- .../package_parser/cli/_run_annotations.py | 6 +++--- package-parser/package_parser/cli/_run_api.py | 2 +- .../annotations/_generate_annotations.py | 6 +++--- .../annotations/_generate_boundary_annotations.py | 4 ++-- .../annotations/_generate_enum_annotations.py | 4 ++-- .../_generate_parameter_importance_annotations.py | 6 +++--- .../annotations/_generate_remove_annotations.py | 6 +++--- .../annotations/_usages_preprocessor.py | 4 ++-- .../annotations/model}/__init__.py | 0 .../annotations/model}/_annotations.py | 0 .../package_parser/processing/api/_ast_visitor.py | 5 +++-- .../package_parser/processing/api/_get_api.py | 2 +- .../processing/api/_get_parameter_list.py | 2 +- ...trategy.py => _AbstractDocumentationParser.py} | 2 +- .../api/documentation/_NumpyDocParser.py | 4 ++-- .../processing/api/documentation/__init__.py | 3 ++- .../{ => processing/api}/model/__init__.py | 0 .../{ => processing/api}/model/api/__init__.py | 0 .../{ => processing/api}/model/api/_api.py | 15 +++++++-------- .../api}/model/api/_parameter_dependencies.py | 2 +- .../{ => processing/api}/model/api/_types.py | 0 .../processing/dependencies/_get_dependency.py | 2 +- .../processing/usages/_ast_visitor.py | 2 +- .../processing/usages/_find_usages.py | 2 +- .../usages/model}/__init__.py | 0 .../usages => processing/usages/model}/_usages.py | 0 .../{ => annotations/model}/test_annotations.py | 2 +- .../annotations}/test_generate_annotations.py | 4 ++-- .../get_api => processing/api}/test_boundaries.py | 2 +- .../get_api => processing/api}/test_enums.py | 2 +- .../api}/test_file_filters.py | 0 .../get_api => processing/api}/test_types.py | 2 +- .../dependencies}/test_get_dependency.py | 2 +- .../dependencies}/test_preprocess_docstring.py | 0 .../{ => processing/usages}/model/test_usages.py | 2 +- 35 files changed, 48 insertions(+), 47 deletions(-) rename package-parser/package_parser/{model/annotations => processing/annotations/model}/__init__.py (100%) rename package-parser/package_parser/{model/annotations => processing/annotations/model}/_annotations.py (100%) rename package-parser/package_parser/processing/api/documentation/{_AbstractDocumentationParsingStrategy.py => _AbstractDocumentationParser.py} (94%) rename package-parser/package_parser/{ => processing/api}/model/__init__.py (100%) rename package-parser/package_parser/{ => processing/api}/model/api/__init__.py (100%) rename package-parser/package_parser/{ => processing/api}/model/api/_api.py (97%) rename package-parser/package_parser/{ => processing/api}/model/api/_parameter_dependencies.py (97%) rename package-parser/package_parser/{ => processing/api}/model/api/_types.py (100%) rename package-parser/package_parser/{model/usages => processing/usages/model}/__init__.py (100%) rename package-parser/package_parser/{model/usages => processing/usages/model}/_usages.py (100%) rename package-parser/tests/model/{ => annotations/model}/test_annotations.py (99%) rename package-parser/tests/{commands/generate_annotations => processing/annotations}/test_generate_annotations.py (91%) rename package-parser/tests/{commands/get_api => processing/api}/test_boundaries.py (97%) rename package-parser/tests/{commands/get_api => processing/api}/test_enums.py (96%) rename package-parser/tests/{commands/get_api => processing/api}/test_file_filters.py (100%) rename package-parser/tests/{commands/get_api => processing/api}/test_types.py (98%) rename package-parser/tests/{commands/get_dependencies => processing/dependencies}/test_get_dependency.py (98%) rename package-parser/tests/{commands/get_dependencies => processing/dependencies}/test_preprocess_docstring.py (100%) rename package-parser/tests/{ => processing/usages}/model/test_usages.py (99%) diff --git a/package-parser/package_parser/cli/_run_annotations.py b/package-parser/package_parser/cli/_run_annotations.py index 6d6c9e62b..02c3e2cd3 100644 --- a/package-parser/package_parser/cli/_run_annotations.py +++ b/package-parser/package_parser/cli/_run_annotations.py @@ -1,9 +1,9 @@ import json from pathlib import Path -from package_parser.model.annotations import AnnotationStore -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore +from package_parser.processing.annotations.model import AnnotationStore +from package_parser.processing.api.model.api import API +from package_parser.processing.usages.model import UsageCountStore from package_parser.processing.annotations import generate_annotations from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/cli/_run_api.py b/package-parser/package_parser/cli/_run_api.py index cc962efcc..cce681065 100644 --- a/package-parser/package_parser/cli/_run_api.py +++ b/package-parser/package_parser/cli/_run_api.py @@ -4,7 +4,7 @@ from package_parser.cli._json_encoder import CustomEncoder from package_parser.cli._shared_constants import _API_KEY -from package_parser.model.api import API +from package_parser.processing.api.model.api import API from package_parser.processing.api import get_api from package_parser.processing.dependencies import get_dependencies from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/processing/annotations/_generate_annotations.py b/package-parser/package_parser/processing/annotations/_generate_annotations.py index b401a6468..3691a1781 100644 --- a/package-parser/package_parser/processing/annotations/_generate_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_annotations.py @@ -1,6 +1,6 @@ -from package_parser.model.annotations import AnnotationStore -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore +from package_parser.processing.annotations.model import AnnotationStore +from package_parser.processing.api.model.api import API +from package_parser.processing.usages.model import UsageCountStore from package_parser.processing.annotations._generate_boundary_annotations import ( _generate_boundary_annotations, ) diff --git a/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py b/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py index f1118c50f..7407c8e48 100644 --- a/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py @@ -1,9 +1,9 @@ -from package_parser.model.annotations import ( +from package_parser.processing.annotations.model import ( AnnotationStore, BoundaryAnnotation, Interval, ) -from package_parser.model.api import API +from package_parser.processing.api.model.api import API from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py index 52b44659e..61c365f46 100644 --- a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py @@ -1,7 +1,7 @@ import re -from package_parser.model.annotations import AnnotationStore, EnumAnnotation, EnumPair -from package_parser.model.api import API +from package_parser.processing.annotations.model import AnnotationStore, EnumAnnotation, EnumPair +from package_parser.processing.api.model.api import API from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py b/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py index 500393d79..43119c29f 100644 --- a/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py @@ -1,13 +1,13 @@ from typing import Any, Optional -from package_parser.model.annotations import ( +from package_parser.processing.annotations.model import ( AnnotationStore, ConstantAnnotation, OptionalAnnotation, RequiredAnnotation, ) -from package_parser.model.api import API, Parameter -from package_parser.model.usages import UsageCountStore +from package_parser.processing.api.model.api import API, Parameter +from package_parser.processing.usages.model import UsageCountStore from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py index 3c7da97a3..eb12a15b6 100644 --- a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py @@ -1,6 +1,6 @@ -from package_parser.model.annotations import AnnotationStore, RemoveAnnotation -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore +from package_parser.processing.annotations.model import AnnotationStore, RemoveAnnotation +from package_parser.processing.api.model.api import API +from package_parser.processing.usages.model import UsageCountStore from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_usages_preprocessor.py b/package-parser/package_parser/processing/annotations/_usages_preprocessor.py index 698e1b189..cbef337be 100644 --- a/package-parser/package_parser/processing/annotations/_usages_preprocessor.py +++ b/package-parser/package_parser/processing/annotations/_usages_preprocessor.py @@ -1,5 +1,5 @@ -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore +from package_parser.processing.api.model.api import API +from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import parent_id diff --git a/package-parser/package_parser/model/annotations/__init__.py b/package-parser/package_parser/processing/annotations/model/__init__.py similarity index 100% rename from package-parser/package_parser/model/annotations/__init__.py rename to package-parser/package_parser/processing/annotations/model/__init__.py diff --git a/package-parser/package_parser/model/annotations/_annotations.py b/package-parser/package_parser/processing/annotations/model/_annotations.py similarity index 100% rename from package-parser/package_parser/model/annotations/_annotations.py rename to package-parser/package_parser/processing/annotations/model/_annotations.py diff --git a/package-parser/package_parser/processing/api/_ast_visitor.py b/package-parser/package_parser/processing/api/_ast_visitor.py index 3396fa6bc..e26d4b3b5 100644 --- a/package-parser/package_parser/processing/api/_ast_visitor.py +++ b/package-parser/package_parser/processing/api/_ast_visitor.py @@ -8,7 +8,7 @@ from astroid.helpers import safe_infer from numpydoc.docscrape import NumpyDocString -from package_parser.model.api import ( +from package_parser.processing.api.model.api import ( API, Class, FromImport, @@ -19,10 +19,11 @@ from package_parser.utils import parent_qualified_name from ._file_filters import _is_init_file from ._get_parameter_list import _get_parameter_list +from .documentation import AbstractDocumentationParser class _AstVisitor: - def __init__(self, api: API) -> None: + def __init__(self, api: API, documentation_parser: AbstractDocumentationParser) -> None: self.reexported: dict[str, list[str]] = {} self.api: API = api self.__declaration_stack: list[Union[Module, Class, Function]] = [] diff --git a/package-parser/package_parser/processing/api/_get_api.py b/package-parser/package_parser/processing/api/_get_api.py index f5c8c1393..30b15d582 100644 --- a/package-parser/package_parser/processing/api/_get_api.py +++ b/package-parser/package_parser/processing/api/_get_api.py @@ -3,7 +3,7 @@ from typing import Optional import astroid -from package_parser.model.api import API +from package_parser.processing.api.model.api import API from package_parser.utils import ASTWalker from ._ast_visitor import _AstVisitor diff --git a/package-parser/package_parser/processing/api/_get_parameter_list.py b/package-parser/package_parser/processing/api/_get_parameter_list.py index 52ca0ce4a..07a9b2511 100644 --- a/package-parser/package_parser/processing/api/_get_parameter_list.py +++ b/package-parser/package_parser/processing/api/_get_parameter_list.py @@ -4,7 +4,7 @@ import astroid from numpydoc.docscrape import NumpyDocString -from package_parser.model.api import Parameter, ParameterAssignment, ParameterAndResultDocstring +from package_parser.processing.api.model.api import Parameter, ParameterAssignment, ParameterAndResultDocstring def _get_parameter_list( diff --git a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py similarity index 94% rename from package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py rename to package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py index be291bed4..08d116b54 100644 --- a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParsingStrategy.py +++ b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py @@ -6,7 +6,7 @@ import astroid -class AbstractDocumentationParsingStrategy(ABC): +class AbstractDocumentationParser(ABC): @abstractmethod def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py index 561304a37..b85995201 100644 --- a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -4,12 +4,12 @@ import numpydoc.docscrape from numpydoc.docscrape import NumpyDocString -from ._AbstractDocumentationParsingStrategy import AbstractDocumentationParsingStrategy, ParameterDocumentation, \ +from ._AbstractDocumentationParser import AbstractDocumentationParser, ParameterDocumentation, \ FunctionDocumentation, ClassDocumentation from ._get_full_docstring import get_full_docstring -class NumpyDocParser(AbstractDocumentationParsingStrategy): +class NumpyDocParser(AbstractDocumentationParser): def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: docstring = get_full_docstring(class_node) diff --git a/package-parser/package_parser/processing/api/documentation/__init__.py b/package-parser/package_parser/processing/api/documentation/__init__.py index ae818e85e..2a5d432bd 100644 --- a/package-parser/package_parser/processing/api/documentation/__init__.py +++ b/package-parser/package_parser/processing/api/documentation/__init__.py @@ -1,3 +1,4 @@ -from ._AbstractDocumentationParsingStrategy import ClassDocumentation, FunctionDocumentation, ParameterDocumentation +from ._AbstractDocumentationParser import AbstractDocumentationParser, ClassDocumentation, FunctionDocumentation, \ + ParameterDocumentation from ._NumpyDocParser import NumpyDocParser from ._get_full_docstring import get_full_docstring diff --git a/package-parser/package_parser/model/__init__.py b/package-parser/package_parser/processing/api/model/__init__.py similarity index 100% rename from package-parser/package_parser/model/__init__.py rename to package-parser/package_parser/processing/api/model/__init__.py diff --git a/package-parser/package_parser/model/api/__init__.py b/package-parser/package_parser/processing/api/model/api/__init__.py similarity index 100% rename from package-parser/package_parser/model/api/__init__.py rename to package-parser/package_parser/processing/api/model/api/__init__.py diff --git a/package-parser/package_parser/model/api/_api.py b/package-parser/package_parser/processing/api/model/api/_api.py similarity index 97% rename from package-parser/package_parser/model/api/_api.py rename to package-parser/package_parser/processing/api/model/api/_api.py index 97f5fab35..4cc6309fe 100644 --- a/package-parser/package_parser/model/api/_api.py +++ b/package-parser/package_parser/processing/api/model/api/_api.py @@ -5,7 +5,8 @@ from enum import Enum from typing import Any, Optional -from package_parser.model.api._types import ( +from package_parser.processing.api.documentation import FunctionDocumentation +from package_parser.processing.api.model.api._types import ( AbstractType, BoundaryType, EnumType, @@ -225,8 +226,7 @@ def __init__( superclasses: list[str], is_public: bool, reexported_by: list[str], - description: str, - docstring: str, + documentation: FunctionDocumentation ) -> None: self.id: str = id_ self.qname: str = qname @@ -269,8 +269,7 @@ class Function: results: list[Result] is_public: bool reexported_by: list[str] - description: str - docstring: str + documentation: FunctionDocumentation @staticmethod def from_json(json: Any) -> Function: @@ -413,7 +412,7 @@ def __init__( default_value: Optional[str], assigned_by: ParameterAssignment, is_public: bool, - docstring: ParameterAndResultDocstring, + documentation: ParameterAndResultDocstring, ) -> None: self.id: str = id_ self.name: str = name @@ -421,8 +420,8 @@ def __init__( self.default_value: Optional[str] = default_value self.assigned_by: ParameterAssignment = assigned_by self.is_public: bool = is_public - self.docstring = docstring - self.type: Type = Type(docstring) + self.docstring = documentation + self.type: Type = Type(documentation) def is_optional(self) -> bool: return self.default_value is not None diff --git a/package-parser/package_parser/model/api/_parameter_dependencies.py b/package-parser/package_parser/processing/api/model/api/_parameter_dependencies.py similarity index 97% rename from package-parser/package_parser/model/api/_parameter_dependencies.py rename to package-parser/package_parser/processing/api/model/api/_parameter_dependencies.py index 417cd7b44..d38c84757 100644 --- a/package-parser/package_parser/model/api/_parameter_dependencies.py +++ b/package-parser/package_parser/processing/api/model/api/_parameter_dependencies.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Any, Dict -from package_parser.model.api import Parameter +from package_parser.processing.api.model.api import Parameter @dataclass diff --git a/package-parser/package_parser/model/api/_types.py b/package-parser/package_parser/processing/api/model/api/_types.py similarity index 100% rename from package-parser/package_parser/model/api/_types.py rename to package-parser/package_parser/processing/api/model/api/_types.py diff --git a/package-parser/package_parser/processing/dependencies/_get_dependency.py b/package-parser/package_parser/processing/dependencies/_get_dependency.py index 08f4b4c22..78f8c00d5 100644 --- a/package-parser/package_parser/processing/dependencies/_get_dependency.py +++ b/package-parser/package_parser/processing/dependencies/_get_dependency.py @@ -1,7 +1,7 @@ from typing import Dict, List, Tuple, Union import spacy -from package_parser.model.api import ( +from package_parser.processing.api.model.api import ( API, Action, APIDependencies, diff --git a/package-parser/package_parser/processing/usages/_ast_visitor.py b/package-parser/package_parser/processing/usages/_ast_visitor.py index 3df798290..d6961500e 100644 --- a/package-parser/package_parser/processing/usages/_ast_visitor.py +++ b/package-parser/package_parser/processing/usages/_ast_visitor.py @@ -4,7 +4,7 @@ import astroid from astroid.arguments import CallSite from astroid.helpers import safe_infer -from package_parser.model.usages import UsageCountStore +from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import parent_id diff --git a/package-parser/package_parser/processing/usages/_find_usages.py b/package-parser/package_parser/processing/usages/_find_usages.py index 7ad946565..c999c24a8 100644 --- a/package-parser/package_parser/processing/usages/_find_usages.py +++ b/package-parser/package_parser/processing/usages/_find_usages.py @@ -8,7 +8,7 @@ from astroid.builder import AstroidBuilder from package_parser.utils import ASTWalker, list_files, parse_python_code -from ...model.usages import UsageCountStore +from package_parser.processing.usages.model import UsageCountStore from ._ast_visitor import _UsageFinder diff --git a/package-parser/package_parser/model/usages/__init__.py b/package-parser/package_parser/processing/usages/model/__init__.py similarity index 100% rename from package-parser/package_parser/model/usages/__init__.py rename to package-parser/package_parser/processing/usages/model/__init__.py diff --git a/package-parser/package_parser/model/usages/_usages.py b/package-parser/package_parser/processing/usages/model/_usages.py similarity index 100% rename from package-parser/package_parser/model/usages/_usages.py rename to package-parser/package_parser/processing/usages/model/_usages.py diff --git a/package-parser/tests/model/test_annotations.py b/package-parser/tests/model/annotations/model/test_annotations.py similarity index 99% rename from package-parser/tests/model/test_annotations.py rename to package-parser/tests/model/annotations/model/test_annotations.py index b7011bd78..b217eeb84 100644 --- a/package-parser/tests/model/test_annotations.py +++ b/package-parser/tests/model/annotations/model/test_annotations.py @@ -1,4 +1,4 @@ -from package_parser.model.annotations import ( +from package_parser.processing.annotations.model import ( ANNOTATION_SCHEMA_VERSION, AbstractAnnotation, AnnotationStore, diff --git a/package-parser/tests/commands/generate_annotations/test_generate_annotations.py b/package-parser/tests/processing/annotations/test_generate_annotations.py similarity index 91% rename from package-parser/tests/commands/generate_annotations/test_generate_annotations.py rename to package-parser/tests/processing/annotations/test_generate_annotations.py index 6cbf01832..9c2622d94 100644 --- a/package-parser/tests/commands/generate_annotations/test_generate_annotations.py +++ b/package-parser/tests/processing/annotations/test_generate_annotations.py @@ -2,8 +2,8 @@ import os import pytest -from package_parser.model.api import API -from package_parser.model.usages import UsageCountStore +from package_parser.processing.api.model.api import API +from package_parser.processing.usages.model import UsageCountStore from package_parser.processing.annotations import generate_annotations diff --git a/package-parser/tests/commands/get_api/test_boundaries.py b/package-parser/tests/processing/api/test_boundaries.py similarity index 97% rename from package-parser/tests/commands/get_api/test_boundaries.py rename to package-parser/tests/processing/api/test_boundaries.py index a6f5439b8..079577ab5 100644 --- a/package-parser/tests/commands/get_api/test_boundaries.py +++ b/package-parser/tests/processing/api/test_boundaries.py @@ -1,5 +1,5 @@ import pytest -from package_parser.model.api._types import BoundaryType +from package_parser.processing.api.model.api import BoundaryType @pytest.mark.parametrize( diff --git a/package-parser/tests/commands/get_api/test_enums.py b/package-parser/tests/processing/api/test_enums.py similarity index 96% rename from package-parser/tests/commands/get_api/test_enums.py rename to package-parser/tests/processing/api/test_enums.py index 1b9d9c8e1..a0e8f0737 100644 --- a/package-parser/tests/commands/get_api/test_enums.py +++ b/package-parser/tests/processing/api/test_enums.py @@ -1,7 +1,7 @@ from typing import Optional import pytest -from package_parser.model.api._types import EnumType +from package_parser.processing.api.model.api import EnumType @pytest.mark.parametrize( diff --git a/package-parser/tests/commands/get_api/test_file_filters.py b/package-parser/tests/processing/api/test_file_filters.py similarity index 100% rename from package-parser/tests/commands/get_api/test_file_filters.py rename to package-parser/tests/processing/api/test_file_filters.py diff --git a/package-parser/tests/commands/get_api/test_types.py b/package-parser/tests/processing/api/test_types.py similarity index 98% rename from package-parser/tests/commands/get_api/test_types.py rename to package-parser/tests/processing/api/test_types.py index d87589c29..9952c0f32 100644 --- a/package-parser/tests/commands/get_api/test_types.py +++ b/package-parser/tests/processing/api/test_types.py @@ -1,7 +1,7 @@ from typing import Any import pytest -from package_parser.model.api import ParameterAndResultDocstring, Type +from package_parser.processing.api.model.api import ParameterAndResultDocstring, Type @pytest.mark.parametrize( diff --git a/package-parser/tests/commands/get_dependencies/test_get_dependency.py b/package-parser/tests/processing/dependencies/test_get_dependency.py similarity index 98% rename from package-parser/tests/commands/get_dependencies/test_get_dependency.py rename to package-parser/tests/processing/dependencies/test_get_dependency.py index e4bd29835..ba82d805a 100644 --- a/package-parser/tests/commands/get_dependencies/test_get_dependency.py +++ b/package-parser/tests/processing/dependencies/test_get_dependency.py @@ -1,5 +1,5 @@ import spacy -from package_parser.model.api import ( +from package_parser.processing.api.model.api import ( Action, Condition, Dependency, diff --git a/package-parser/tests/commands/get_dependencies/test_preprocess_docstring.py b/package-parser/tests/processing/dependencies/test_preprocess_docstring.py similarity index 100% rename from package-parser/tests/commands/get_dependencies/test_preprocess_docstring.py rename to package-parser/tests/processing/dependencies/test_preprocess_docstring.py diff --git a/package-parser/tests/model/test_usages.py b/package-parser/tests/processing/usages/model/test_usages.py similarity index 99% rename from package-parser/tests/model/test_usages.py rename to package-parser/tests/processing/usages/model/test_usages.py index be4079139..be20779c6 100644 --- a/package-parser/tests/model/test_usages.py +++ b/package-parser/tests/processing/usages/model/test_usages.py @@ -1,7 +1,7 @@ from typing import Any import pytest -from package_parser.model.usages import USAGES_SCHEMA_VERSION, UsageCountStore +from package_parser.processing.usages.model import USAGES_SCHEMA_VERSION, UsageCountStore @pytest.fixture From cefef31a0131bd91d94d6d559b5f156da08fcfc0 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 13:12:18 +0200 Subject: [PATCH 05/16] refactor(parser): update API model --- .../_AbstractDocumentationParser.py | 32 +++++++++++++++ .../processing/api/model/api/_api.py | 40 ++++++++++--------- .../dependencies/test_get_dependency.py | 15 ++++--- 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py index 08d116b54..4b8aa9339 100644 --- a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py +++ b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses from abc import ABC, abstractmethod from dataclasses import dataclass @@ -30,15 +31,46 @@ class ClassDocumentation: description: str full_docstring: str + @staticmethod + def from_dict(d: dict) -> ClassDocumentation: + return ClassDocumentation( + description=d.get("description", ""), + full_docstring=d.get("full_docstring", ""), + ) + + def to_dict(self): + return dataclasses.asdict(self) + @dataclass class FunctionDocumentation: description: str full_docstring: str + @staticmethod + def from_dict(d: dict) -> FunctionDocumentation: + return FunctionDocumentation( + description=d.get("description", ""), + full_docstring=d.get("full_docstring", ""), + ) + + def to_dict(self): + return dataclasses.asdict(self) + @dataclass class ParameterDocumentation: type: str default_value: str description: str + + @staticmethod + def from_dict(d: dict) -> ParameterDocumentation: + return ParameterDocumentation( + type=d.get("type", ""), + default_value=d.get("default_value", ""), + description=d.get("description", ""), + ) + + def to_dict(self): + dataclasses.asdict(self) diff --git a/package-parser/package_parser/processing/api/model/api/_api.py b/package-parser/package_parser/processing/api/model/api/_api.py index 4cc6309fe..ae2e6aaf9 100644 --- a/package-parser/package_parser/processing/api/model/api/_api.py +++ b/package-parser/package_parser/processing/api/model/api/_api.py @@ -5,7 +5,8 @@ from enum import Enum from typing import Any, Optional -from package_parser.processing.api.documentation import FunctionDocumentation +from package_parser.processing.api.documentation import FunctionDocumentation, ClassDocumentation, \ + ParameterDocumentation from package_parser.processing.api.model.api._types import ( AbstractType, BoundaryType, @@ -209,8 +210,10 @@ def from_json(json: Any) -> Class: json.get("superclasses", []), json.get("is_public", True), json.get("reexported_by", []), - json.get("description", ""), - json.get("docstring", ""), + ClassDocumentation( + description=json.get("description", ""), + full_docstring=json.get("docstring", ""), + ) ) for method_id in json["methods"]: @@ -226,7 +229,7 @@ def __init__( superclasses: list[str], is_public: bool, reexported_by: list[str], - documentation: FunctionDocumentation + documentation: ClassDocumentation ) -> None: self.id: str = id_ self.qname: str = qname @@ -235,8 +238,7 @@ def __init__( self.methods: list[str] = [] self.is_public: bool = is_public self.reexported_by: list[str] = reexported_by - self.description: str = description - self.docstring: str = docstring + self.documentation: ClassDocumentation = documentation @property def name(self) -> str: @@ -255,8 +257,8 @@ def to_json(self) -> Any: "methods": self.methods, "is_public": self.is_public, "reexported_by": self.reexported_by, - "description": self.description, - "docstring": self.docstring, + "description": self.documentation.description, + "docstring": self.documentation.full_docstring, } @@ -284,8 +286,10 @@ def from_json(json: Any) -> Function: [Result.from_json(result_json) for result_json in json.get("results", [])], json.get("is_public", True), json.get("reexported_by", []), - json.get("description", ""), - json.get("docstring", ""), + FunctionDocumentation( + description=json.get("description", ""), + full_docstring=json.get("docstring", ""), + ) ) @property @@ -302,21 +306,21 @@ def to_json(self) -> Any: "results": [result.to_json() for result in self.results], "is_public": self.is_public, "reexported_by": self.reexported_by, - "description": self.description, - "docstring": self.docstring, + "description": self.documentation.description, + "docstring": self.documentation.full_docstring, } class Type: def __init__( self, - typestring: ParameterAndResultDocstring, + typestring: ParameterDocumentation, ) -> None: self.type: Optional[AbstractType] = Type.create_type(typestring) @classmethod def create_type( - cls, docstring: ParameterAndResultDocstring + cls, docstring: ParameterDocumentation ) -> Optional[AbstractType]: type_string = docstring.type types: list[AbstractType] = list() @@ -401,7 +405,7 @@ def from_json(json: Any): json.get("default_value", None), ParameterAssignment[json.get("assigned_by", "POSITION_OR_NAME")], json.get("is_public", True), - ParameterAndResultDocstring.from_json(json.get("docstring", {})), + ParameterDocumentation.from_dict(json.get("docstring", {})), ) def __init__( @@ -412,7 +416,7 @@ def __init__( default_value: Optional[str], assigned_by: ParameterAssignment, is_public: bool, - documentation: ParameterAndResultDocstring, + documentation: ParameterDocumentation, ) -> None: self.id: str = id_ self.name: str = name @@ -420,7 +424,7 @@ def __init__( self.default_value: Optional[str] = default_value self.assigned_by: ParameterAssignment = assigned_by self.is_public: bool = is_public - self.docstring = documentation + self.documentation = documentation self.type: Type = Type(documentation) def is_optional(self) -> bool: @@ -437,7 +441,7 @@ def to_json(self) -> Any: "default_value": self.default_value, "assigned_by": self.assigned_by.name, "is_public": self.is_public, - "docstring": self.docstring.to_json(), + "docstring": self.documentation.to_dict(), "type": self.type.to_json(), } diff --git a/package-parser/tests/processing/dependencies/test_get_dependency.py b/package-parser/tests/processing/dependencies/test_get_dependency.py index ba82d805a..42536f529 100644 --- a/package-parser/tests/processing/dependencies/test_get_dependency.py +++ b/package-parser/tests/processing/dependencies/test_get_dependency.py @@ -1,10 +1,11 @@ import spacy + +from package_parser.processing.api.documentation import ParameterDocumentation from package_parser.processing.api.model.api import ( Action, Condition, Dependency, Parameter, - ParameterAndResultDocstring, ParameterAssignment, ParameterHasValue, ParameterIsIgnored, @@ -103,8 +104,10 @@ def test_extract_dependencies_from_docstring_pattern_adverbial_clause(): default_value=None, assigned_by=ParameterAssignment.NAME_ONLY, is_public=True, - docstring=ParameterAndResultDocstring( - type="param possible types", description=param_docstring_nlp.text + documentation=ParameterDocumentation( + type="param possible types", + default_value="", + description=param_docstring_nlp.text ), ) dependee_param = Parameter( @@ -114,8 +117,10 @@ def test_extract_dependencies_from_docstring_pattern_adverbial_clause(): default_value=None, assigned_by=ParameterAssignment.NAME_ONLY, is_public=True, - docstring=ParameterAndResultDocstring( - type="param possible types", description="param probability docstring" + documentation=ParameterDocumentation( + type="param possible types", + default_value="", + description="param probability docstring" ), ) func_params = [dependent_param, dependee_param] From 0befb057b85b272a9aca93fc44bfb536f065b855 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 14:10:05 +0200 Subject: [PATCH 06/16] refactor(parser): use injected documentation parser for class and function documentation --- .../processing/api/_ast_visitor.py | 63 +++++++------------ .../package_parser/processing/api/_get_api.py | 4 +- .../processing/api/_get_parameter_list.py | 9 +-- 3 files changed, 28 insertions(+), 48 deletions(-) diff --git a/package-parser/package_parser/processing/api/_ast_visitor.py b/package-parser/package_parser/processing/api/_ast_visitor.py index e26d4b3b5..172da61f8 100644 --- a/package-parser/package_parser/processing/api/_ast_visitor.py +++ b/package-parser/package_parser/processing/api/_ast_visitor.py @@ -1,4 +1,3 @@ -import inspect import logging import re from typing import Optional, Union @@ -6,7 +5,6 @@ import astroid from astroid.context import InferenceContext from astroid.helpers import safe_infer -from numpydoc.docscrape import NumpyDocString from package_parser.processing.api.model.api import ( API, @@ -23,7 +21,8 @@ class _AstVisitor: - def __init__(self, api: API, documentation_parser: AbstractDocumentationParser) -> None: + def __init__(self, documentation_parser: AbstractDocumentationParser, api: API) -> None: + self.documentation_parser: AbstractDocumentationParser = documentation_parser self.reexported: dict[str, list[str]] = {} self.api: API = api self.__declaration_stack: list[Union[Module, Class, Function]] = [] @@ -139,18 +138,15 @@ def enter_classdef(self, class_node: astroid.ClassDef) -> None: else: decorator_names = [] - numpydoc = NumpyDocString(inspect.cleandoc(class_node.doc or "")) - # Remember class, so we can later add methods class_ = Class( - self.__get_id(class_node.name), - qname, - decorator_names, - class_node.basenames, - self.is_public(class_node.name, qname), - self.reexported.get(qname, []), - _AstVisitor.__description(numpydoc), - class_node.doc, + id_=self.__get_id(class_node.name), + qname=qname, + decorators=decorator_names, + superclasses=class_node.basenames, + is_public=self.is_public(class_node.name, qname), + reexported_by=self.reexported.get(qname, []), + documentation=self.documentation_parser.get_class_documentation(class_node) ) self.__declaration_stack.append(class_) @@ -176,21 +172,23 @@ def enter_functiondef(self, function_node: astroid.FunctionDef) -> None: else: decorator_names = [] - numpydoc = NumpyDocString(inspect.cleandoc(function_node.doc or "")) is_public = self.is_public(function_node.name, qname) function = Function( - self.__get_function_id(function_node.name, decorator_names), - qname, - decorator_names, - _get_parameter_list( - function_node, self.__get_id(function_node.name), qname, is_public, + id=self.__get_function_id(function_node.name, decorator_names), + qname=qname, + decorators=decorator_names, + parameters=_get_parameter_list( + self.documentation_parser, + function_node, + self.__get_id(function_node.name), + qname, + is_public ), - [], # TODO: results - is_public, - self.reexported.get(qname, []), - _AstVisitor.__description(numpydoc), - function_node.doc, + results=[], # TODO: results + is_public=is_public, + reexported_by=self.reexported.get(qname, []), + documentation=self.documentation_parser.get_function_documentation(function_node) ) self.__declaration_stack.append(function) @@ -210,23 +208,6 @@ def leave_functiondef(self, _: astroid.FunctionDef) -> None: self.api.add_function(function) parent.add_method(function.id) - @staticmethod - def __description(numpydoc: NumpyDocString) -> str: - has_summary = "Summary" in numpydoc and len(numpydoc["Summary"]) > 0 - has_extended_summary = ( - "Extended Summary" in numpydoc and len(numpydoc["Extended Summary"]) > 0 - ) - - result = "" - if has_summary: - result += "\n".join(numpydoc["Summary"]) - if has_summary and has_extended_summary: - result += "\n\n" - if has_extended_summary: - result += "\n".join(numpydoc["Extended Summary"]) - return result - - def is_public(self, name: str, qualified_name: str) -> bool: if name.startswith("_") and not name.endswith("__"): return False diff --git a/package-parser/package_parser/processing/api/_get_api.py b/package-parser/package_parser/processing/api/_get_api.py index 30b15d582..4a52e395f 100644 --- a/package-parser/package_parser/processing/api/_get_api.py +++ b/package-parser/package_parser/processing/api/_get_api.py @@ -14,6 +14,7 @@ package_files, package_root, ) +from .documentation import NumpyDocParser def get_api(package_name: str, root: Optional[Path] = None) -> API: @@ -24,7 +25,8 @@ def get_api(package_name: str, root: Optional[Path] = None) -> API: files = package_files(root) api = API(dist, package_name, dist_version) - callable_visitor = _AstVisitor(api) + documentation_parser = NumpyDocParser() + callable_visitor = _AstVisitor(documentation_parser, api) walker = ASTWalker(callable_visitor) for file in files: diff --git a/package-parser/package_parser/processing/api/_get_parameter_list.py b/package-parser/package_parser/processing/api/_get_parameter_list.py index 07a9b2511..12869886d 100644 --- a/package-parser/package_parser/processing/api/_get_parameter_list.py +++ b/package-parser/package_parser/processing/api/_get_parameter_list.py @@ -4,15 +4,12 @@ import astroid from numpydoc.docscrape import NumpyDocString +from package_parser.processing.api.documentation import AbstractDocumentationParser from package_parser.processing.api.model.api import Parameter, ParameterAssignment, ParameterAndResultDocstring -def _get_parameter_list( - node: astroid.FunctionDef, - function_id: str, - function_qname: str, - function_is_public: bool, -) -> list[Parameter]: +def _get_parameter_list(documentation_parser: AbstractDocumentationParser, node: astroid.FunctionDef, function_id: str, + function_qname: str, function_is_public: bool) -> list[Parameter]: parameters = node.args n_implicit_parameters = node.implicit_parameters() From 411e8e2615ff7b0f67544db14b6b00e6d6b50ccf Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 14:25:15 +0200 Subject: [PATCH 07/16] refactor(parser): remove references to now outdated code --- .../processing/api/_get_parameter_list.py | 72 +++++++------------ .../processing/api/model/api/__init__.py | 2 +- .../processing/api/model/api/_api.py | 12 ++-- .../annotations/model/test_annotations.py | 0 .../api/documentation/test_NumpyDocParser.py | 24 ++++++- .../documentation/test_get_full_docstring.py | 8 +++ .../tests/processing/api/test_types.py | 17 +++-- 7 files changed, 74 insertions(+), 61 deletions(-) rename package-parser/tests/{model => processing}/annotations/model/test_annotations.py (100%) diff --git a/package-parser/package_parser/processing/api/_get_parameter_list.py b/package-parser/package_parser/processing/api/_get_parameter_list.py index 12869886d..7ce39c09a 100644 --- a/package-parser/package_parser/processing/api/_get_parameter_list.py +++ b/package-parser/package_parser/processing/api/_get_parameter_list.py @@ -1,24 +1,20 @@ -import inspect from typing import Optional import astroid -from numpydoc.docscrape import NumpyDocString from package_parser.processing.api.documentation import AbstractDocumentationParser -from package_parser.processing.api.model.api import Parameter, ParameterAssignment, ParameterAndResultDocstring +from package_parser.processing.api.model.api import Parameter, ParameterAssignment -def _get_parameter_list(documentation_parser: AbstractDocumentationParser, node: astroid.FunctionDef, function_id: str, - function_qname: str, function_is_public: bool) -> list[Parameter]: - parameters = node.args - n_implicit_parameters = node.implicit_parameters() - - # For constructors (__init__ functions) the parameters are described on the class - if node.name == "__init__" and isinstance(node.parent, astroid.ClassDef): - docstring = node.parent.doc - else: - docstring = node.doc - function_numpydoc = NumpyDocString(inspect.cleandoc(docstring or "")) +def _get_parameter_list( + documentation_parser: AbstractDocumentationParser, + function_node: astroid.FunctionDef, + function_id: str, + function_qname: str, + function_is_public: bool +) -> list[Parameter]: + parameters = function_node.args + n_implicit_parameters = function_node.implicit_parameters() # Arguments that can be specified positionally only ( f(1) works but not f(x=1) ) result = [ @@ -29,7 +25,7 @@ def _get_parameter_list(documentation_parser: AbstractDocumentationParser, node: default_value=None, assigned_by=ParameterAssignment.POSITION_ONLY, is_public=function_is_public, - docstring=_get_parameter_docstring(function_numpydoc, it.name), + documentation=documentation_parser.get_parameter_documentation(function_node, it.name) ) for it in parameters.posonlyargs ] @@ -37,16 +33,16 @@ def _get_parameter_list(documentation_parser: AbstractDocumentationParser, node: # Arguments that can be specified positionally or by name ( f(1) and f(x=1) both work ) result += [ Parameter( - function_id + "/" + it.name, - it.name, - function_qname + "." + it.name, - _get_parameter_default( + id_=function_id + "/" + it.name, + name=it.name, + qname=function_qname + "." + it.name, + default_value=_get_parameter_default( parameters.defaults, index - len(parameters.args) + len(parameters.defaults), ), - ParameterAssignment.POSITION_OR_NAME, - function_is_public, - _get_parameter_docstring(function_numpydoc, it.name), + assigned_by=ParameterAssignment.POSITION_OR_NAME, + is_public=function_is_public, + documentation=documentation_parser.get_parameter_documentation(function_node, it.name) ) for index, it in enumerate(parameters.args) ] @@ -54,16 +50,16 @@ def _get_parameter_list(documentation_parser: AbstractDocumentationParser, node: # Arguments that can be specified by name only ( f(x=1) works but not f(1) ) result += [ Parameter( - function_id + "/" + it.name, - it.name, - function_qname + "." + it.name, - _get_parameter_default( + id_=function_id + "/" + it.name, + name=it.name, + qname=function_qname + "." + it.name, + default_value=_get_parameter_default( parameters.kw_defaults, index - len(parameters.kwonlyargs) + len(parameters.kw_defaults), ), - ParameterAssignment.NAME_ONLY, - function_is_public, - _get_parameter_docstring(function_numpydoc, it.name), + assigned_by=ParameterAssignment.NAME_ONLY, + is_public=function_is_public, + documentation=documentation_parser.get_parameter_documentation(function_node, it.name) ) for index, it in enumerate(parameters.kwonlyargs) ] @@ -86,21 +82,3 @@ def _get_parameter_default( return default.as_string() else: return None - - -def _get_parameter_docstring( - function_numpydoc: NumpyDocString, - parameter_name: str -) -> ParameterAndResultDocstring: - parameters_numpydoc = function_numpydoc["Parameters"] - candidate_parameters_numpydoc = [ - it for it in parameters_numpydoc if it.name == parameter_name - ] - - if len(candidate_parameters_numpydoc) > 0: - last_parameter_numpydoc = candidate_parameters_numpydoc[-1] - return ParameterAndResultDocstring( - last_parameter_numpydoc.type, "\n".join(last_parameter_numpydoc.desc) - ) - - return ParameterAndResultDocstring("", "") diff --git a/package-parser/package_parser/processing/api/model/api/__init__.py b/package-parser/package_parser/processing/api/model/api/__init__.py index 32007cdd3..3a4b65097 100644 --- a/package-parser/package_parser/processing/api/model/api/__init__.py +++ b/package-parser/package_parser/processing/api/model/api/__init__.py @@ -7,7 +7,7 @@ Import, Module, Parameter, - ParameterAndResultDocstring, + ResultDocstring, ParameterAssignment, Result, Type, diff --git a/package-parser/package_parser/processing/api/model/api/_api.py b/package-parser/package_parser/processing/api/model/api/_api.py index ae2e6aaf9..8f16b0d93 100644 --- a/package-parser/package_parser/processing/api/model/api/_api.py +++ b/package-parser/package_parser/processing/api/model/api/_api.py @@ -314,9 +314,9 @@ def to_json(self) -> Any: class Type: def __init__( self, - typestring: ParameterDocumentation, + parameter_documentation: ParameterDocumentation, ) -> None: - self.type: Optional[AbstractType] = Type.create_type(typestring) + self.type: Optional[AbstractType] = Type.create_type(parameter_documentation) @classmethod def create_type( @@ -456,13 +456,13 @@ class ParameterAssignment(Enum): @dataclass class Result: name: str - docstring: ParameterAndResultDocstring + docstring: ResultDocstring @staticmethod def from_json(json: Any) -> Result: return Result( json["name"], - ParameterAndResultDocstring.from_json(json.get("docstring", {})), + ResultDocstring.from_json(json.get("docstring", {})), ) def to_json(self) -> Any: @@ -470,13 +470,13 @@ def to_json(self) -> Any: @dataclass -class ParameterAndResultDocstring: +class ResultDocstring: type: str description: str @staticmethod def from_json(json: Any): - return ParameterAndResultDocstring( + return ResultDocstring( json.get("type", ""), json.get("description", ""), ) diff --git a/package-parser/tests/model/annotations/model/test_annotations.py b/package-parser/tests/processing/annotations/model/test_annotations.py similarity index 100% rename from package-parser/tests/model/annotations/model/test_annotations.py rename to package-parser/tests/processing/annotations/model/test_annotations.py diff --git a/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py index 80e3d7ec5..d5ea2b5ad 100644 --- a/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py +++ b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py @@ -41,6 +41,10 @@ class C: description="", full_docstring="" )), + ], + ids=[ + "class with documentation", + "class without documentation", ] ) def test_get_class_documentation( @@ -83,7 +87,12 @@ def f(): (function_without_documentation, FunctionDocumentation( description="", full_docstring="" - )), + )) + + ], + ids=[ + "function with documentation", + "function without documentation", ] ) def test_get_function_documentation( @@ -213,6 +222,19 @@ def f( default_value="", description="" )), + ], + ids=[ + "existing class parameter", + "missing class parameter", + "function parameter with no type and no default", + "function parameter with type and no default", + "function parameter with optional unknown default", + "function parameter with default syntax 1 (just space)", + "function parameter with default syntax 2 (colon)", + "function parameter with default syntax 3 (equals)", + "function parameter with grouped parameters 1", + "function parameter with grouped parameters 2", + "missing function parameter", ] ) def test_get_parameter_documentation( diff --git a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py index cad48d9c9..b5e749f74 100644 --- a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py +++ b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py @@ -65,6 +65,14 @@ def f(): (function_with_multi_line_documentation, "Lorem ipsum.\n\nDolor sit amet."), (function_with_single_line_documentation, "Lorem ipsum."), (function_without_documentation, ""), + ], + ids=[ + "class with multi line documentation", + "class with single line documentation", + "class without documentation", + "function with multi line documentation", + "function with single line documentation", + "function without documentation", ] ) def test_get_full_docstring(python_code: str, expected_docstring: str): diff --git a/package-parser/tests/processing/api/test_types.py b/package-parser/tests/processing/api/test_types.py index 9952c0f32..04d650d3c 100644 --- a/package-parser/tests/processing/api/test_types.py +++ b/package-parser/tests/processing/api/test_types.py @@ -1,7 +1,9 @@ from typing import Any import pytest -from package_parser.processing.api.model.api import ParameterAndResultDocstring, Type + +from package_parser.processing.api.documentation import ParameterDocumentation +from package_parser.processing.api.model.api import Type @pytest.mark.parametrize( @@ -62,7 +64,7 @@ { "kind": "NamedType", "name": "shape (n_samples, n_classes) or (n_samples, 1) when " - "binary.", + "binary.", }, ], }, @@ -70,7 +72,7 @@ ], ) def test_union_from_string(docstring_type: str, expected: dict[str, Any]): - result = Type(ParameterAndResultDocstring(docstring_type, "")) + result = Type(ParameterDocumentation(docstring_type, "", "")) assert result.to_json() == expected @@ -103,7 +105,7 @@ def test_union_from_string(docstring_type: str, expected: dict[str, Any]): ], ) def test_boundary_from_string(docstring_type: str, expected: dict[str, Any]): - assert Type(ParameterAndResultDocstring("", docstring_type)).to_json() == expected + assert Type(ParameterDocumentation("", "", docstring_type)).to_json() == expected @pytest.mark.parametrize( @@ -135,8 +137,11 @@ def test_boundary_and_union_from_string( docstring_type: str, docstring_description: str, expected: dict[str, Any] ): assert ( - Type( - ParameterAndResultDocstring(docstring_type, docstring_description) + Type(ParameterDocumentation( + type=docstring_type, + default_value="", + description=docstring_description + ) ).to_json() == expected ) From 5725672d60b391dd98aa6dafd439ee75ce535c6e Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 14:43:24 +0200 Subject: [PATCH 08/16] refactor(parser): minor changes --- .../documentation/_APIElementDocumentation.py | 44 ++++++++++++++ .../_AbstractDocumentationParser.py | 58 ++----------------- .../api/documentation/_NumpyDocParser.py | 4 +- .../processing/api/documentation/__init__.py | 4 +- .../test_APIElementDocumentation.py | 37 ++++++++++++ 5 files changed, 91 insertions(+), 56 deletions(-) create mode 100644 package-parser/package_parser/processing/api/documentation/_APIElementDocumentation.py create mode 100644 package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py diff --git a/package-parser/package_parser/processing/api/documentation/_APIElementDocumentation.py b/package-parser/package_parser/processing/api/documentation/_APIElementDocumentation.py new file mode 100644 index 000000000..47103135a --- /dev/null +++ b/package-parser/package_parser/processing/api/documentation/_APIElementDocumentation.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass + + +@dataclass +class ClassDocumentation: + description: str = "" + full_docstring: str = "" + + @staticmethod + def from_dict(d: dict) -> ClassDocumentation: + return ClassDocumentation(**d) + + def to_dict(self): + return dataclasses.asdict(self) + + +@dataclass +class FunctionDocumentation: + description: str = "" + full_docstring: str = "" + + @staticmethod + def from_dict(d: dict) -> FunctionDocumentation: + return FunctionDocumentation(**d) + + def to_dict(self): + return dataclasses.asdict(self) + + +@dataclass +class ParameterDocumentation: + type: str = "" + default_value: str = "" + description: str = "" + + @staticmethod + def from_dict(d: dict) -> ParameterDocumentation: + return ParameterDocumentation(**d) + + def to_dict(self): + return dataclasses.asdict(self) diff --git a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py index 4b8aa9339..0334726ed 100644 --- a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py +++ b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py @@ -1,11 +1,15 @@ from __future__ import annotations -import dataclasses from abc import ABC, abstractmethod -from dataclasses import dataclass import astroid +from ._APIElementDocumentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation +) + class AbstractDocumentationParser(ABC): @@ -24,53 +28,3 @@ def get_parameter_documentation( parameter_name: str ) -> ParameterDocumentation: pass - - -@dataclass -class ClassDocumentation: - description: str - full_docstring: str - - @staticmethod - def from_dict(d: dict) -> ClassDocumentation: - return ClassDocumentation( - description=d.get("description", ""), - full_docstring=d.get("full_docstring", ""), - ) - - def to_dict(self): - return dataclasses.asdict(self) - - -@dataclass -class FunctionDocumentation: - description: str - full_docstring: str - - @staticmethod - def from_dict(d: dict) -> FunctionDocumentation: - return FunctionDocumentation( - description=d.get("description", ""), - full_docstring=d.get("full_docstring", ""), - ) - - def to_dict(self): - return dataclasses.asdict(self) - - -@dataclass -class ParameterDocumentation: - type: str - default_value: str - description: str - - @staticmethod - def from_dict(d: dict) -> ParameterDocumentation: - return ParameterDocumentation( - type=d.get("type", ""), - default_value=d.get("default_value", ""), - description=d.get("description", ""), - ) - - def to_dict(self): - dataclasses.asdict(self) diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py index b85995201..84c68b15b 100644 --- a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -4,8 +4,8 @@ import numpydoc.docscrape from numpydoc.docscrape import NumpyDocString -from ._AbstractDocumentationParser import AbstractDocumentationParser, ParameterDocumentation, \ - FunctionDocumentation, ClassDocumentation +from ._AbstractDocumentationParser import AbstractDocumentationParser +from ._APIElementDocumentation import ClassDocumentation, FunctionDocumentation, ParameterDocumentation from ._get_full_docstring import get_full_docstring diff --git a/package-parser/package_parser/processing/api/documentation/__init__.py b/package-parser/package_parser/processing/api/documentation/__init__.py index 2a5d432bd..fc534a257 100644 --- a/package-parser/package_parser/processing/api/documentation/__init__.py +++ b/package-parser/package_parser/processing/api/documentation/__init__.py @@ -1,4 +1,4 @@ -from ._AbstractDocumentationParser import AbstractDocumentationParser, ClassDocumentation, FunctionDocumentation, \ - ParameterDocumentation +from ._AbstractDocumentationParser import AbstractDocumentationParser +from ._APIElementDocumentation import ClassDocumentation, FunctionDocumentation, ParameterDocumentation from ._NumpyDocParser import NumpyDocParser from ._get_full_docstring import get_full_docstring diff --git a/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py b/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py new file mode 100644 index 000000000..3881e0caf --- /dev/null +++ b/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py @@ -0,0 +1,37 @@ +import pytest + +from package_parser.processing.api.documentation import ClassDocumentation, FunctionDocumentation, \ + ParameterDocumentation + + +@pytest.mark.parametrize( + "class_documentation", + [ + ClassDocumentation(), + ClassDocumentation(description="foo", full_docstring="foo bar"), + ] +) +def test_dict_conversion_for_class_documentation(class_documentation: ClassDocumentation): + assert ClassDocumentation.from_dict(class_documentation.to_dict()) == class_documentation + + +@pytest.mark.parametrize( + "function_documentation", + [ + FunctionDocumentation(), + FunctionDocumentation(description="foo", full_docstring="foo bar"), + ] +) +def test_dict_conversion_for_function_documentation(function_documentation: FunctionDocumentation): + assert FunctionDocumentation.from_dict(function_documentation.to_dict()) == function_documentation + + +@pytest.mark.parametrize( + "parameter_documentation", + [ + ParameterDocumentation(), + ParameterDocumentation(type="int", default_value="1", description="foo bar"), + ] +) +def test_dict_conversion_for_parameter_documentation(parameter_documentation: ParameterDocumentation): + assert ParameterDocumentation.from_dict(parameter_documentation.to_dict()) == parameter_documentation From 9a65a5fe4388a724bb253d0a91119230ef8010ae Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 14:51:28 +0200 Subject: [PATCH 09/16] fix(parser): crash --- .../package_parser/processing/dependencies/_get_dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-parser/package_parser/processing/dependencies/_get_dependency.py b/package-parser/package_parser/processing/dependencies/_get_dependency.py index 78f8c00d5..8f59c260c 100644 --- a/package-parser/package_parser/processing/dependencies/_get_dependency.py +++ b/package-parser/package_parser/processing/dependencies/_get_dependency.py @@ -196,7 +196,7 @@ def get_dependencies(api: API) -> APIDependencies: parameters = function.parameters all_dependencies[function_name] = {} for parameter in parameters: - docstring = parameter.docstring.description + docstring = parameter.documentation.description docstring_preprocessed = preprocess_docstring(docstring) doc = nlp(docstring_preprocessed) param_dependencies = [] From 58f29fbca8058351c7f2c84094dbf041d4cdd74d Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 15:29:37 +0200 Subject: [PATCH 10/16] perf(parser): improve performance of numpydoc parser --- .../api/documentation/_NumpyDocParser.py | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py index 84c68b15b..f88344be3 100644 --- a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -10,11 +10,22 @@ class NumpyDocParser(AbstractDocumentationParser): + """ + Parses documentation in the NumpyDoc format. See https://numpydoc.readthedocs.io/en/latest/format.html for more + information. + + This class is not thread-safe. Each thread should create its own instance. + """ + + def __init__(self): + self.__cached_function_node: astroid.FunctionDef | None = None + self.__cached_numpydoc_string: NumpyDocString | None = None + def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: docstring = get_full_docstring(class_node) return ClassDocumentation( - description=_get_description(docstring), + description=_get_description(NumpyDocString(docstring)), full_docstring=docstring ) @@ -22,7 +33,12 @@ def get_function_documentation(self, function_node: astroid.FunctionDef) -> Func docstring = get_full_docstring(function_node) return FunctionDocumentation( - description=_get_description(docstring), + description=_get_description( + self.__get_cached_function_numpydoc_string( + function_node, + docstring + ) + ), full_docstring=docstring ) @@ -40,7 +56,7 @@ def get_parameter_documentation( # Find matching parameter docstrings. Numpydoc allows multiple parameters to be documented at once. See # https://numpydoc.readthedocs.io/en/latest/format.html#parameters for more information. - function_numpydoc = NumpyDocString(docstring) + function_numpydoc = self.__get_cached_function_numpydoc_string(function_node, docstring) all_parameters_numpydoc: list[numpydoc.docscrape.Parameter] = function_numpydoc.get("Parameters", []) matching_parameters_numpydoc = [ it for it in all_parameters_numpydoc @@ -55,23 +71,42 @@ def get_parameter_documentation( ) last_parameter_numpydoc = matching_parameters_numpydoc[-1] - type, default_value = _get_type_and_default_value(last_parameter_numpydoc) + type_, default_value = _get_type_and_default_value(last_parameter_numpydoc) return ParameterDocumentation( - type=type, + type=type_, default_value=default_value, description="\n".join([line.strip() for line in last_parameter_numpydoc.desc]) ) + def __get_cached_function_numpydoc_string( + self, + function_node: astroid.FunctionDef, + docstring: str + ) -> NumpyDocString: + """ + Returns the NumpyDocString for the given function node. It is only recomputed when the function node differs + from the previous one that was passed to this function. This avoids reparsing the docstring for the function + itself and all of its parameters. + + On Lars's system this caused a significant performance improvement: Previously, 8.382s were spent inside the + function get_parameter_documentation when parsing sklearn. Afterwards, it was only 2.113s. + """ + + if self.__cached_function_node is not function_node: + self.__cached_function_node = function_node + self.__cached_numpydoc_string = NumpyDocString(docstring) + + return self.__cached_numpydoc_string + -def _get_description(docstring: str) -> str: +def _get_description(numpydoc_string: NumpyDocString) -> str: """ Returns the concatenated summary and extended summary parts of the given docstring or an empty string if these parts are blank. """ - numpydoc_ = NumpyDocString(docstring) - summary: list[str] = numpydoc_.get("Summary", []) - extended_summary: list[str] = numpydoc_.get("Extended Summary", []) + summary: list[str] = numpydoc_string.get("Summary", []) + extended_summary: list[str] = numpydoc_string.get("Extended Summary", []) result = "" result += "\n".join([line.strip() for line in summary]) From 323a5608013bb34931765c8525c5516aee1593b4 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 15:52:37 +0200 Subject: [PATCH 11/16] refactor(parser): further restructuring --- .../package_parser/cli/_run_annotations.py | 2 +- package-parser/package_parser/cli/_run_api.py | 2 +- .../annotations/_generate_annotations.py | 2 +- .../_generate_boundary_annotations.py | 2 +- .../annotations/_generate_enum_annotations.py | 2 +- ...nerate_parameter_importance_annotations.py | 2 +- .../_generate_remove_annotations.py | 2 +- .../annotations/_usages_preprocessor.py | 2 +- .../processing/api/_ast_visitor.py | 2 +- .../package_parser/processing/api/_get_api.py | 2 +- .../processing/api/_get_parameter_list.py | 2 +- .../api/documentation/_get_full_docstring.py | 3 ++- .../processing/api/model/__init__.py | 15 +++++++++++ .../processing/api/model/{api => }/_api.py | 2 +- .../processing/api/model/{api => }/_types.py | 5 ++-- .../processing/api/model/api/__init__.py | 27 ------------------- .../processing/dependencies/__init__.py | 13 +++++++++ .../dependencies/_get_dependency.py | 6 +++-- .../_parameter_dependencies.py | 2 +- .../annotations/test_generate_annotations.py | 2 +- .../tests/processing/api/test_boundaries.py | 2 +- .../tests/processing/api/test_enums.py | 2 +- .../tests/processing/api/test_types.py | 2 +- .../dependencies/test_get_dependency.py | 8 +++--- 24 files changed, 59 insertions(+), 52 deletions(-) rename package-parser/package_parser/processing/api/model/{api => }/_api.py (99%) rename package-parser/package_parser/processing/api/model/{api => }/_types.py (97%) delete mode 100644 package-parser/package_parser/processing/api/model/api/__init__.py rename package-parser/package_parser/processing/{api/model/api => dependencies}/_parameter_dependencies.py (97%) diff --git a/package-parser/package_parser/cli/_run_annotations.py b/package-parser/package_parser/cli/_run_annotations.py index 02c3e2cd3..c651c3f86 100644 --- a/package-parser/package_parser/cli/_run_annotations.py +++ b/package-parser/package_parser/cli/_run_annotations.py @@ -2,7 +2,7 @@ from pathlib import Path from package_parser.processing.annotations.model import AnnotationStore -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore from package_parser.processing.annotations import generate_annotations from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/cli/_run_api.py b/package-parser/package_parser/cli/_run_api.py index cce681065..0231114ee 100644 --- a/package-parser/package_parser/cli/_run_api.py +++ b/package-parser/package_parser/cli/_run_api.py @@ -4,7 +4,7 @@ from package_parser.cli._json_encoder import CustomEncoder from package_parser.cli._shared_constants import _API_KEY -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from package_parser.processing.api import get_api from package_parser.processing.dependencies import get_dependencies from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/processing/annotations/_generate_annotations.py b/package-parser/package_parser/processing/annotations/_generate_annotations.py index 3691a1781..fe5452f4a 100644 --- a/package-parser/package_parser/processing/annotations/_generate_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_annotations.py @@ -1,5 +1,5 @@ from package_parser.processing.annotations.model import AnnotationStore -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore from package_parser.processing.annotations._generate_boundary_annotations import ( _generate_boundary_annotations, diff --git a/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py b/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py index 7407c8e48..57b2f0d50 100644 --- a/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_boundary_annotations.py @@ -3,7 +3,7 @@ BoundaryAnnotation, Interval, ) -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py index 61c365f46..a6c3666bb 100644 --- a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py @@ -1,7 +1,7 @@ import re from package_parser.processing.annotations.model import AnnotationStore, EnumAnnotation, EnumPair -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py b/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py index 43119c29f..b26ef9bd9 100644 --- a/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_parameter_importance_annotations.py @@ -6,7 +6,7 @@ OptionalAnnotation, RequiredAnnotation, ) -from package_parser.processing.api.model.api import API, Parameter +from package_parser.processing.api.model import API, Parameter from package_parser.processing.usages.model import UsageCountStore from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py index eb12a15b6..d80adfe21 100644 --- a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py @@ -1,5 +1,5 @@ from package_parser.processing.annotations.model import AnnotationStore, RemoveAnnotation -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_usages_preprocessor.py b/package-parser/package_parser/processing/annotations/_usages_preprocessor.py index cbef337be..7b17f0b8e 100644 --- a/package-parser/package_parser/processing/annotations/_usages_preprocessor.py +++ b/package-parser/package_parser/processing/annotations/_usages_preprocessor.py @@ -1,4 +1,4 @@ -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import parent_id diff --git a/package-parser/package_parser/processing/api/_ast_visitor.py b/package-parser/package_parser/processing/api/_ast_visitor.py index 172da61f8..aee5615f0 100644 --- a/package-parser/package_parser/processing/api/_ast_visitor.py +++ b/package-parser/package_parser/processing/api/_ast_visitor.py @@ -6,7 +6,7 @@ from astroid.context import InferenceContext from astroid.helpers import safe_infer -from package_parser.processing.api.model.api import ( +from package_parser.processing.api.model import ( API, Class, FromImport, diff --git a/package-parser/package_parser/processing/api/_get_api.py b/package-parser/package_parser/processing/api/_get_api.py index 4a52e395f..5060fc6bf 100644 --- a/package-parser/package_parser/processing/api/_get_api.py +++ b/package-parser/package_parser/processing/api/_get_api.py @@ -3,7 +3,7 @@ from typing import Optional import astroid -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from package_parser.utils import ASTWalker from ._ast_visitor import _AstVisitor diff --git a/package-parser/package_parser/processing/api/_get_parameter_list.py b/package-parser/package_parser/processing/api/_get_parameter_list.py index 7ce39c09a..4b2267c3d 100644 --- a/package-parser/package_parser/processing/api/_get_parameter_list.py +++ b/package-parser/package_parser/processing/api/_get_parameter_list.py @@ -3,7 +3,7 @@ import astroid from package_parser.processing.api.documentation import AbstractDocumentationParser -from package_parser.processing.api.model.api import Parameter, ParameterAssignment +from package_parser.processing.api.model import Parameter, ParameterAssignment def _get_parameter_list( diff --git a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py index 9269c28a9..e0d0e81b2 100644 --- a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py +++ b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py @@ -1,9 +1,10 @@ import inspect +from typing import Union import astroid -def get_full_docstring(declaration: astroid.ClassDef | astroid.FunctionDef) -> str: +def get_full_docstring(declaration: Union[astroid.ClassDef, astroid.FunctionDef]) -> str: """ Returns the full docstring of the given declaration or an empty string if no docstring is available. Indentation is cleaned up. diff --git a/package-parser/package_parser/processing/api/model/__init__.py b/package-parser/package_parser/processing/api/model/__init__.py index e69de29bb..345d22c90 100644 --- a/package-parser/package_parser/processing/api/model/__init__.py +++ b/package-parser/package_parser/processing/api/model/__init__.py @@ -0,0 +1,15 @@ +from ._api import ( + API, + API_SCHEMA_VERSION, + Class, + FromImport, + Function, + Import, + Module, + Parameter, + ResultDocstring, + ParameterAssignment, + Result, + Type, +) +from ._types import AbstractType, BoundaryType, EnumType, NamedType, UnionType diff --git a/package-parser/package_parser/processing/api/model/api/_api.py b/package-parser/package_parser/processing/api/model/_api.py similarity index 99% rename from package-parser/package_parser/processing/api/model/api/_api.py rename to package-parser/package_parser/processing/api/model/_api.py index 8f16b0d93..4eae71721 100644 --- a/package-parser/package_parser/processing/api/model/api/_api.py +++ b/package-parser/package_parser/processing/api/model/_api.py @@ -7,7 +7,7 @@ from package_parser.processing.api.documentation import FunctionDocumentation, ClassDocumentation, \ ParameterDocumentation -from package_parser.processing.api.model.api._types import ( +from ._types import ( AbstractType, BoundaryType, EnumType, diff --git a/package-parser/package_parser/processing/api/model/api/_types.py b/package-parser/package_parser/processing/api/model/_types.py similarity index 97% rename from package-parser/package_parser/processing/api/model/api/_types.py rename to package-parser/package_parser/processing/api/model/_types.py index 266cec275..5c12e04e4 100644 --- a/package-parser/package_parser/processing/api/model/api/_types.py +++ b/package-parser/package_parser/processing/api/model/_types.py @@ -45,10 +45,10 @@ def remove_backslash(e: str): curr_quote = None for i, char in enumerate(enum_str): if char in quotes and (i == 0 or (i > 0 and enum_str[i - 1] != "\\")): - if inside_value == False: + if not inside_value: inside_value = True curr_quote = char - elif inside_value == True: + elif inside_value: if curr_quote == char: inside_value = False curr_quote = None @@ -92,6 +92,7 @@ def _is_inclusive(cls, bracket: str) -> bool: @classmethod def from_string(cls, string: str) -> Optional[BoundaryType]: + # language=PythonRegExp pattern = r"""(?Pfloat|int)?[ ] # optional base type of either float or int (in|of)[ ](the[ ])?(range|interval)[ ](of[ ])? # 'in' or 'of', optional 'the', 'range' or 'interval', optional 'of' `?(?P[\[(])(?P[-+]?\d+(.\d*)?|negative_infinity),[ ] # left side of the range diff --git a/package-parser/package_parser/processing/api/model/api/__init__.py b/package-parser/package_parser/processing/api/model/api/__init__.py deleted file mode 100644 index 3a4b65097..000000000 --- a/package-parser/package_parser/processing/api/model/api/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from ._api import ( - API, - API_SCHEMA_VERSION, - Class, - FromImport, - Function, - Import, - Module, - Parameter, - ResultDocstring, - ParameterAssignment, - Result, - Type, -) -from ._parameter_dependencies import ( - Action, - APIDependencies, - Condition, - Dependency, - ParameterHasValue, - ParameterIsIgnored, - ParameterIsIllegal, - ParameterIsNone, - RuntimeCondition, - StaticCondition, -) -from ._types import AbstractType, BoundaryType, EnumType, NamedType, UnionType diff --git a/package-parser/package_parser/processing/dependencies/__init__.py b/package-parser/package_parser/processing/dependencies/__init__.py index ab7fc933e..63eda82bd 100644 --- a/package-parser/package_parser/processing/dependencies/__init__.py +++ b/package-parser/package_parser/processing/dependencies/__init__.py @@ -5,3 +5,16 @@ extract_lefts_and_rights, get_dependencies, ) +from ._parameter_dependencies import ( + Action, + RuntimeAction, + StaticAction, + Condition, + RuntimeCondition, + StaticCondition, + Dependency, + ParameterHasValue, + ParameterIsIgnored, + ParameterIsIllegal, + ParameterIsNone, +) diff --git a/package-parser/package_parser/processing/dependencies/_get_dependency.py b/package-parser/package_parser/processing/dependencies/_get_dependency.py index 8f59c260c..5dd4143c7 100644 --- a/package-parser/package_parser/processing/dependencies/_get_dependency.py +++ b/package-parser/package_parser/processing/dependencies/_get_dependency.py @@ -1,13 +1,15 @@ from typing import Dict, List, Tuple, Union import spacy -from package_parser.processing.api.model.api import ( +from package_parser.processing.api.model import ( API, + Parameter, +) +from ._parameter_dependencies import ( Action, APIDependencies, Condition, Dependency, - Parameter, ParameterHasValue, ParameterIsIgnored, ParameterIsIllegal, diff --git a/package-parser/package_parser/processing/api/model/api/_parameter_dependencies.py b/package-parser/package_parser/processing/dependencies/_parameter_dependencies.py similarity index 97% rename from package-parser/package_parser/processing/api/model/api/_parameter_dependencies.py rename to package-parser/package_parser/processing/dependencies/_parameter_dependencies.py index d38c84757..55570f338 100644 --- a/package-parser/package_parser/processing/api/model/api/_parameter_dependencies.py +++ b/package-parser/package_parser/processing/dependencies/_parameter_dependencies.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Any, Dict -from package_parser.processing.api.model.api import Parameter +from package_parser.processing.api.model import Parameter @dataclass diff --git a/package-parser/tests/processing/annotations/test_generate_annotations.py b/package-parser/tests/processing/annotations/test_generate_annotations.py index 9c2622d94..b00aa188e 100644 --- a/package-parser/tests/processing/annotations/test_generate_annotations.py +++ b/package-parser/tests/processing/annotations/test_generate_annotations.py @@ -2,7 +2,7 @@ import os import pytest -from package_parser.processing.api.model.api import API +from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore from package_parser.processing.annotations import generate_annotations diff --git a/package-parser/tests/processing/api/test_boundaries.py b/package-parser/tests/processing/api/test_boundaries.py index 079577ab5..7001faad3 100644 --- a/package-parser/tests/processing/api/test_boundaries.py +++ b/package-parser/tests/processing/api/test_boundaries.py @@ -1,5 +1,5 @@ import pytest -from package_parser.processing.api.model.api import BoundaryType +from package_parser.processing.api.model import BoundaryType @pytest.mark.parametrize( diff --git a/package-parser/tests/processing/api/test_enums.py b/package-parser/tests/processing/api/test_enums.py index a0e8f0737..3a37b06a1 100644 --- a/package-parser/tests/processing/api/test_enums.py +++ b/package-parser/tests/processing/api/test_enums.py @@ -1,7 +1,7 @@ from typing import Optional import pytest -from package_parser.processing.api.model.api import EnumType +from package_parser.processing.api.model import EnumType @pytest.mark.parametrize( diff --git a/package-parser/tests/processing/api/test_types.py b/package-parser/tests/processing/api/test_types.py index 04d650d3c..3a0a6b3bf 100644 --- a/package-parser/tests/processing/api/test_types.py +++ b/package-parser/tests/processing/api/test_types.py @@ -3,7 +3,7 @@ import pytest from package_parser.processing.api.documentation import ParameterDocumentation -from package_parser.processing.api.model.api import Type +from package_parser.processing.api.model import Type @pytest.mark.parametrize( diff --git a/package-parser/tests/processing/dependencies/test_get_dependency.py b/package-parser/tests/processing/dependencies/test_get_dependency.py index 42536f529..155288a39 100644 --- a/package-parser/tests/processing/dependencies/test_get_dependency.py +++ b/package-parser/tests/processing/dependencies/test_get_dependency.py @@ -1,12 +1,14 @@ import spacy from package_parser.processing.api.documentation import ParameterDocumentation -from package_parser.processing.api.model.api import ( +from package_parser.processing.api.model import ( + Parameter, + ParameterAssignment +) +from package_parser.processing.dependencies import ( Action, Condition, Dependency, - Parameter, - ParameterAssignment, ParameterHasValue, ParameterIsIgnored, ParameterIsIllegal, From 3dca4cecce5591dfb1a3f84340f7b8eda4024a59 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 15:54:38 +0200 Subject: [PATCH 12/16] build: set mypy python version to 3.10 --- .../processing/api/documentation/_get_full_docstring.py | 3 +-- package-parser/pyproject.toml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py index e0d0e81b2..9269c28a9 100644 --- a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py +++ b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py @@ -1,10 +1,9 @@ import inspect -from typing import Union import astroid -def get_full_docstring(declaration: Union[astroid.ClassDef, astroid.FunctionDef]) -> str: +def get_full_docstring(declaration: astroid.ClassDef | astroid.FunctionDef) -> str: """ Returns the full docstring of the given declaration or an empty string if no docstring is available. Indentation is cleaned up. diff --git a/package-parser/pyproject.toml b/package-parser/pyproject.toml index ab7343473..61383cbc6 100644 --- a/package-parser/pyproject.toml +++ b/package-parser/pyproject.toml @@ -23,6 +23,7 @@ pytest = "^7.1.2" pytest-cov = "^3.0.0" [tool.mypy] +python_version = "3.10" ignore_missing_imports = true disallow_untyped-calls = true disallow_untyped-defs = true From 3135f156f5c97181565d0e3736114f98d01ada62 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 15:55:34 +0200 Subject: [PATCH 13/16] build: set mypy python version to 3.10 --- package-parser/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/package-parser/pyproject.toml b/package-parser/pyproject.toml index 61383cbc6..d378fa9ba 100644 --- a/package-parser/pyproject.toml +++ b/package-parser/pyproject.toml @@ -24,6 +24,7 @@ pytest-cov = "^3.0.0" [tool.mypy] python_version = "3.10" +no_site_packages = true ignore_missing_imports = true disallow_untyped-calls = true disallow_untyped-defs = true From a3fa83739414ef75c58441238e88630605df7a74 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 16:04:25 +0200 Subject: [PATCH 14/16] style: fix linter errors --- .../processing/api/documentation/_NumpyDocParser.py | 5 +++-- .../api/documentation/_get_full_docstring.py | 3 ++- .../package_parser/processing/api/model/_types.py | 12 ++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py index f88344be3..29165efc3 100644 --- a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -1,4 +1,5 @@ import re +from typing import Optional import astroid import numpydoc.docscrape @@ -18,8 +19,8 @@ class NumpyDocParser(AbstractDocumentationParser): """ def __init__(self): - self.__cached_function_node: astroid.FunctionDef | None = None - self.__cached_numpydoc_string: NumpyDocString | None = None + self.__cached_function_node: Optional[astroid.FunctionDef] = None + self.__cached_numpydoc_string: Optional[NumpyDocString] = None def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: docstring = get_full_docstring(class_node) diff --git a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py index 9269c28a9..e0d0e81b2 100644 --- a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py +++ b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py @@ -1,9 +1,10 @@ import inspect +from typing import Union import astroid -def get_full_docstring(declaration: astroid.ClassDef | astroid.FunctionDef) -> str: +def get_full_docstring(declaration: Union[astroid.ClassDef, astroid.FunctionDef]) -> str: """ Returns the full docstring of the given declaration or an empty string if no docstring is available. Indentation is cleaned up. diff --git a/package-parser/package_parser/processing/api/model/_types.py b/package-parser/package_parser/processing/api/model/_types.py index 5c12e04e4..44bde2c4c 100644 --- a/package-parser/package_parser/processing/api/model/_types.py +++ b/package-parser/package_parser/processing/api/model/_types.py @@ -103,17 +103,21 @@ def from_string(cls, string: str) -> Optional[BoundaryType]: base_type = match.group("base_type") if base_type is None: base_type = "float" - base_type = eval(base_type) + + if base_type == "int": + base_type_converter = int + else: + base_type_converter = float min_value = match.group("min") if min_value != "negative_infinity": - min_value = base_type(min_value) + min_value = base_type_converter(min_value) else: min_value = BoundaryType.NEGATIVE_INFINITY max_value = match.group("max") if max_value != "infinity": - max_value = base_type(max_value) + max_value = base_type_converter(max_value) else: max_value = BoundaryType.INFINITY @@ -123,7 +127,7 @@ def from_string(cls, string: str) -> Optional[BoundaryType]: max_inclusive = BoundaryType._is_inclusive(max_bracket) return BoundaryType( - base_type.__name__, min_value, max_value, min_inclusive, max_inclusive + base_type, min_value, max_value, min_inclusive, max_inclusive ) return None From c4343c16434ca4ba61ff3f463ca3f201785aed5d Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Tue, 28 Jun 2022 16:18:05 +0200 Subject: [PATCH 15/16] style: fix linter errors --- .../api/documentation/_NumpyDocParser.py | 4 ++-- .../processing/api/model/_types.py | 21 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py index 29165efc3..2f848a0ec 100644 --- a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -1,5 +1,5 @@ import re -from typing import Optional +from typing import Optional, Tuple import astroid import numpydoc.docscrape @@ -124,7 +124,7 @@ def _is_matching_parameter_numpydoc(parameter_numpydoc: numpydoc.docscrape.Param return any(name.strip() == parameter_name for name in parameter_numpydoc.name.split(",")) -def _get_type_and_default_value(parameter_numpydoc: numpydoc.docscrape.Parameter) -> (str, str): +def _get_type_and_default_value(parameter_numpydoc: numpydoc.docscrape.Parameter) -> Tuple[str, str]: """ Returns the type and default value for the given NumpyDoc. """ diff --git a/package-parser/package_parser/processing/api/model/_types.py b/package-parser/package_parser/processing/api/model/_types.py index 44bde2c4c..c2ac79cc9 100644 --- a/package-parser/package_parser/processing/api/model/_types.py +++ b/package-parser/package_parser/processing/api/model/_types.py @@ -76,7 +76,7 @@ class BoundaryType(AbstractType): INFINITY: ClassVar = "Infinity" base_type: str - min: Union[float, int] + min: Union[float, int, str] max: Union[float, int, str] min_inclusive: bool max_inclusive: bool @@ -104,20 +104,21 @@ def from_string(cls, string: str) -> Optional[BoundaryType]: if base_type is None: base_type = "float" - if base_type == "int": - base_type_converter = int - else: - base_type_converter = float - - min_value = match.group("min") + min_value: Union[str, int, float] = match.group("min") if min_value != "negative_infinity": - min_value = base_type_converter(min_value) + if base_type == "int": + min_value = int(min_value) + else: + min_value = float(min_value) else: min_value = BoundaryType.NEGATIVE_INFINITY - max_value = match.group("max") + max_value: Union[str, int, float] = match.group("max") if max_value != "infinity": - max_value = base_type_converter(max_value) + if base_type == "int": + max_value = int(max_value) + else: + max_value = float(max_value) else: max_value = BoundaryType.INFINITY From 619caeb56069081689be156822df9b6718c2dd3a Mon Sep 17 00:00:00 2001 From: lars-reimann Date: Tue, 28 Jun 2022 14:22:30 +0000 Subject: [PATCH 16/16] style: apply automatic fixes of linters --- .../package_parser/cli/_run_annotations.py | 2 +- package-parser/package_parser/cli/_run_api.py | 2 +- .../annotations/_generate_annotations.py | 6 +- .../annotations/_generate_enum_annotations.py | 6 +- .../_generate_remove_annotations.py | 5 +- .../processing/api/_ast_visitor.py | 14 +- .../processing/api/_get_parameter_list.py | 18 +- .../_AbstractDocumentationParser.py | 15 +- .../api/documentation/_NumpyDocParser.py | 68 +++--- .../processing/api/documentation/__init__.py | 8 +- .../api/documentation/_get_full_docstring.py | 4 +- .../processing/api/model/__init__.py | 2 +- .../processing/api/model/_api.py | 24 +- .../processing/dependencies/__init__.py | 8 +- .../dependencies/_get_dependency.py | 17 +- .../processing/usages/_find_usages.py | 2 +- .../annotations/test_generate_annotations.py | 2 +- .../test_APIElementDocumentation.py | 41 +++- .../api/documentation/test_NumpyDocParser.py | 225 +++++++++++------- .../documentation/test_get_full_docstring.py | 11 +- .../tests/processing/api/test_types.py | 12 +- .../dependencies/test_get_dependency.py | 14 +- .../processing/usages/model/test_usages.py | 5 +- 23 files changed, 299 insertions(+), 212 deletions(-) diff --git a/package-parser/package_parser/cli/_run_annotations.py b/package-parser/package_parser/cli/_run_annotations.py index c651c3f86..ecb6221c8 100644 --- a/package-parser/package_parser/cli/_run_annotations.py +++ b/package-parser/package_parser/cli/_run_annotations.py @@ -1,10 +1,10 @@ import json from pathlib import Path +from package_parser.processing.annotations import generate_annotations from package_parser.processing.annotations.model import AnnotationStore from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore -from package_parser.processing.annotations import generate_annotations from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/cli/_run_api.py b/package-parser/package_parser/cli/_run_api.py index 0231114ee..a8372d7e6 100644 --- a/package-parser/package_parser/cli/_run_api.py +++ b/package-parser/package_parser/cli/_run_api.py @@ -4,8 +4,8 @@ from package_parser.cli._json_encoder import CustomEncoder from package_parser.cli._shared_constants import _API_KEY -from package_parser.processing.api.model import API from package_parser.processing.api import get_api +from package_parser.processing.api.model import API from package_parser.processing.dependencies import get_dependencies from package_parser.utils import ensure_file_exists diff --git a/package-parser/package_parser/processing/annotations/_generate_annotations.py b/package-parser/package_parser/processing/annotations/_generate_annotations.py index fe5452f4a..b7603854a 100644 --- a/package-parser/package_parser/processing/annotations/_generate_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_annotations.py @@ -1,6 +1,3 @@ -from package_parser.processing.annotations.model import AnnotationStore -from package_parser.processing.api.model import API -from package_parser.processing.usages.model import UsageCountStore from package_parser.processing.annotations._generate_boundary_annotations import ( _generate_boundary_annotations, ) @@ -16,6 +13,9 @@ from package_parser.processing.annotations._usages_preprocessor import ( _preprocess_usages, ) +from package_parser.processing.annotations.model import AnnotationStore +from package_parser.processing.api.model import API +from package_parser.processing.usages.model import UsageCountStore def generate_annotations(api: API, usages: UsageCountStore) -> AnnotationStore: diff --git a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py index a6c3666bb..d5ae11138 100644 --- a/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_enum_annotations.py @@ -1,6 +1,10 @@ import re -from package_parser.processing.annotations.model import AnnotationStore, EnumAnnotation, EnumPair +from package_parser.processing.annotations.model import ( + AnnotationStore, + EnumAnnotation, + EnumPair, +) from package_parser.processing.api.model import API from ._constants import autogen_author diff --git a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py index d80adfe21..0217c6c35 100644 --- a/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py +++ b/package-parser/package_parser/processing/annotations/_generate_remove_annotations.py @@ -1,4 +1,7 @@ -from package_parser.processing.annotations.model import AnnotationStore, RemoveAnnotation +from package_parser.processing.annotations.model import ( + AnnotationStore, + RemoveAnnotation, +) from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore diff --git a/package-parser/package_parser/processing/api/_ast_visitor.py b/package-parser/package_parser/processing/api/_ast_visitor.py index aee5615f0..2b7358243 100644 --- a/package-parser/package_parser/processing/api/_ast_visitor.py +++ b/package-parser/package_parser/processing/api/_ast_visitor.py @@ -5,7 +5,6 @@ import astroid from astroid.context import InferenceContext from astroid.helpers import safe_infer - from package_parser.processing.api.model import ( API, Class, @@ -15,13 +14,16 @@ Module, ) from package_parser.utils import parent_qualified_name + from ._file_filters import _is_init_file from ._get_parameter_list import _get_parameter_list from .documentation import AbstractDocumentationParser class _AstVisitor: - def __init__(self, documentation_parser: AbstractDocumentationParser, api: API) -> None: + def __init__( + self, documentation_parser: AbstractDocumentationParser, api: API + ) -> None: self.documentation_parser: AbstractDocumentationParser = documentation_parser self.reexported: dict[str, list[str]] = {} self.api: API = api @@ -146,7 +148,7 @@ def enter_classdef(self, class_node: astroid.ClassDef) -> None: superclasses=class_node.basenames, is_public=self.is_public(class_node.name, qname), reexported_by=self.reexported.get(qname, []), - documentation=self.documentation_parser.get_class_documentation(class_node) + documentation=self.documentation_parser.get_class_documentation(class_node), ) self.__declaration_stack.append(class_) @@ -183,12 +185,14 @@ def enter_functiondef(self, function_node: astroid.FunctionDef) -> None: function_node, self.__get_id(function_node.name), qname, - is_public + is_public, ), results=[], # TODO: results is_public=is_public, reexported_by=self.reexported.get(qname, []), - documentation=self.documentation_parser.get_function_documentation(function_node) + documentation=self.documentation_parser.get_function_documentation( + function_node + ), ) self.__declaration_stack.append(function) diff --git a/package-parser/package_parser/processing/api/_get_parameter_list.py b/package-parser/package_parser/processing/api/_get_parameter_list.py index 4b2267c3d..34bd1ddd3 100644 --- a/package-parser/package_parser/processing/api/_get_parameter_list.py +++ b/package-parser/package_parser/processing/api/_get_parameter_list.py @@ -1,7 +1,6 @@ from typing import Optional import astroid - from package_parser.processing.api.documentation import AbstractDocumentationParser from package_parser.processing.api.model import Parameter, ParameterAssignment @@ -11,7 +10,7 @@ def _get_parameter_list( function_node: astroid.FunctionDef, function_id: str, function_qname: str, - function_is_public: bool + function_is_public: bool, ) -> list[Parameter]: parameters = function_node.args n_implicit_parameters = function_node.implicit_parameters() @@ -25,7 +24,9 @@ def _get_parameter_list( default_value=None, assigned_by=ParameterAssignment.POSITION_ONLY, is_public=function_is_public, - documentation=documentation_parser.get_parameter_documentation(function_node, it.name) + documentation=documentation_parser.get_parameter_documentation( + function_node, it.name + ), ) for it in parameters.posonlyargs ] @@ -42,7 +43,9 @@ def _get_parameter_list( ), assigned_by=ParameterAssignment.POSITION_OR_NAME, is_public=function_is_public, - documentation=documentation_parser.get_parameter_documentation(function_node, it.name) + documentation=documentation_parser.get_parameter_documentation( + function_node, it.name + ), ) for index, it in enumerate(parameters.args) ] @@ -59,7 +62,9 @@ def _get_parameter_list( ), assigned_by=ParameterAssignment.NAME_ONLY, is_public=function_is_public, - documentation=documentation_parser.get_parameter_documentation(function_node, it.name) + documentation=documentation_parser.get_parameter_documentation( + function_node, it.name + ), ) for index, it in enumerate(parameters.kwonlyargs) ] @@ -72,8 +77,7 @@ def _get_parameter_list( def _get_parameter_default( - defaults: list[astroid.NodeNG], - default_index: int + defaults: list[astroid.NodeNG], default_index: int ) -> Optional[str]: if 0 <= default_index < len(defaults): default = defaults[default_index] diff --git a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py index 0334726ed..8bf0cff66 100644 --- a/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py +++ b/package-parser/package_parser/processing/api/documentation/_AbstractDocumentationParser.py @@ -7,24 +7,25 @@ from ._APIElementDocumentation import ( ClassDocumentation, FunctionDocumentation, - ParameterDocumentation + ParameterDocumentation, ) class AbstractDocumentationParser(ABC): - @abstractmethod - def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: + def get_class_documentation( + self, class_node: astroid.ClassDef + ) -> ClassDocumentation: pass @abstractmethod - def get_function_documentation(self, function_node: astroid.FunctionDef) -> FunctionDocumentation: + def get_function_documentation( + self, function_node: astroid.FunctionDef + ) -> FunctionDocumentation: pass @abstractmethod def get_parameter_documentation( - self, - function_node: astroid.FunctionDef, - parameter_name: str + self, function_node: astroid.FunctionDef, parameter_name: str ) -> ParameterDocumentation: pass diff --git a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py index 2f848a0ec..2c014326c 100644 --- a/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py +++ b/package-parser/package_parser/processing/api/documentation/_NumpyDocParser.py @@ -6,7 +6,11 @@ from numpydoc.docscrape import NumpyDocString from ._AbstractDocumentationParser import AbstractDocumentationParser -from ._APIElementDocumentation import ClassDocumentation, FunctionDocumentation, ParameterDocumentation +from ._APIElementDocumentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, +) from ._get_full_docstring import get_full_docstring @@ -22,67 +26,69 @@ def __init__(self): self.__cached_function_node: Optional[astroid.FunctionDef] = None self.__cached_numpydoc_string: Optional[NumpyDocString] = None - def get_class_documentation(self, class_node: astroid.ClassDef) -> ClassDocumentation: + def get_class_documentation( + self, class_node: astroid.ClassDef + ) -> ClassDocumentation: docstring = get_full_docstring(class_node) return ClassDocumentation( description=_get_description(NumpyDocString(docstring)), - full_docstring=docstring + full_docstring=docstring, ) - def get_function_documentation(self, function_node: astroid.FunctionDef) -> FunctionDocumentation: + def get_function_documentation( + self, function_node: astroid.FunctionDef + ) -> FunctionDocumentation: docstring = get_full_docstring(function_node) return FunctionDocumentation( description=_get_description( - self.__get_cached_function_numpydoc_string( - function_node, - docstring - ) + self.__get_cached_function_numpydoc_string(function_node, docstring) ), - full_docstring=docstring + full_docstring=docstring, ) def get_parameter_documentation( - self, - function_node: astroid.FunctionDef, - parameter_name: str + self, function_node: astroid.FunctionDef, parameter_name: str ) -> ParameterDocumentation: # For constructors (__init__ functions) the parameters are described on the class - if function_node.name == "__init__" and isinstance(function_node.parent, astroid.ClassDef): + if function_node.name == "__init__" and isinstance( + function_node.parent, astroid.ClassDef + ): docstring = get_full_docstring(function_node.parent) else: docstring = get_full_docstring(function_node) # Find matching parameter docstrings. Numpydoc allows multiple parameters to be documented at once. See # https://numpydoc.readthedocs.io/en/latest/format.html#parameters for more information. - function_numpydoc = self.__get_cached_function_numpydoc_string(function_node, docstring) - all_parameters_numpydoc: list[numpydoc.docscrape.Parameter] = function_numpydoc.get("Parameters", []) + function_numpydoc = self.__get_cached_function_numpydoc_string( + function_node, docstring + ) + all_parameters_numpydoc: list[ + numpydoc.docscrape.Parameter + ] = function_numpydoc.get("Parameters", []) matching_parameters_numpydoc = [ - it for it in all_parameters_numpydoc + it + for it in all_parameters_numpydoc if _is_matching_parameter_numpydoc(it, parameter_name) ] if len(matching_parameters_numpydoc) == 0: - return ParameterDocumentation( - type="", - default_value="", - description="" - ) + return ParameterDocumentation(type="", default_value="", description="") last_parameter_numpydoc = matching_parameters_numpydoc[-1] type_, default_value = _get_type_and_default_value(last_parameter_numpydoc) return ParameterDocumentation( type=type_, default_value=default_value, - description="\n".join([line.strip() for line in last_parameter_numpydoc.desc]) + description="\n".join( + [line.strip() for line in last_parameter_numpydoc.desc] + ), ) def __get_cached_function_numpydoc_string( - self, - function_node: astroid.FunctionDef, - docstring: str + self, function_node: astroid.FunctionDef, docstring: str ) -> NumpyDocString: """ Returns the NumpyDocString for the given function node. It is only recomputed when the function node differs @@ -116,15 +122,21 @@ def _get_description(numpydoc_string: NumpyDocString) -> str: return result.strip() -def _is_matching_parameter_numpydoc(parameter_numpydoc: numpydoc.docscrape.Parameter, parameter_name: str) -> bool: +def _is_matching_parameter_numpydoc( + parameter_numpydoc: numpydoc.docscrape.Parameter, parameter_name: str +) -> bool: """ Returns whether the given NumpyDoc applied to the parameter with the given name. """ - return any(name.strip() == parameter_name for name in parameter_numpydoc.name.split(",")) + return any( + name.strip() == parameter_name for name in parameter_numpydoc.name.split(",") + ) -def _get_type_and_default_value(parameter_numpydoc: numpydoc.docscrape.Parameter) -> Tuple[str, str]: +def _get_type_and_default_value( + parameter_numpydoc: numpydoc.docscrape.Parameter, +) -> Tuple[str, str]: """ Returns the type and default value for the given NumpyDoc. """ diff --git a/package-parser/package_parser/processing/api/documentation/__init__.py b/package-parser/package_parser/processing/api/documentation/__init__.py index fc534a257..0342e961c 100644 --- a/package-parser/package_parser/processing/api/documentation/__init__.py +++ b/package-parser/package_parser/processing/api/documentation/__init__.py @@ -1,4 +1,8 @@ from ._AbstractDocumentationParser import AbstractDocumentationParser -from ._APIElementDocumentation import ClassDocumentation, FunctionDocumentation, ParameterDocumentation -from ._NumpyDocParser import NumpyDocParser +from ._APIElementDocumentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, +) from ._get_full_docstring import get_full_docstring +from ._NumpyDocParser import NumpyDocParser diff --git a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py index e0d0e81b2..72d783d78 100644 --- a/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py +++ b/package-parser/package_parser/processing/api/documentation/_get_full_docstring.py @@ -4,7 +4,9 @@ import astroid -def get_full_docstring(declaration: Union[astroid.ClassDef, astroid.FunctionDef]) -> str: +def get_full_docstring( + declaration: Union[astroid.ClassDef, astroid.FunctionDef] +) -> str: """ Returns the full docstring of the given declaration or an empty string if no docstring is available. Indentation is cleaned up. diff --git a/package-parser/package_parser/processing/api/model/__init__.py b/package-parser/package_parser/processing/api/model/__init__.py index 345d22c90..104a58da6 100644 --- a/package-parser/package_parser/processing/api/model/__init__.py +++ b/package-parser/package_parser/processing/api/model/__init__.py @@ -7,9 +7,9 @@ Import, Module, Parameter, - ResultDocstring, ParameterAssignment, Result, + ResultDocstring, Type, ) from ._types import AbstractType, BoundaryType, EnumType, NamedType, UnionType diff --git a/package-parser/package_parser/processing/api/model/_api.py b/package-parser/package_parser/processing/api/model/_api.py index 4eae71721..6e26bc47c 100644 --- a/package-parser/package_parser/processing/api/model/_api.py +++ b/package-parser/package_parser/processing/api/model/_api.py @@ -5,17 +5,15 @@ from enum import Enum from typing import Any, Optional -from package_parser.processing.api.documentation import FunctionDocumentation, ClassDocumentation, \ - ParameterDocumentation -from ._types import ( - AbstractType, - BoundaryType, - EnumType, - NamedType, - UnionType, +from package_parser.processing.api.documentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, ) from package_parser.utils import parent_id +from ._types import AbstractType, BoundaryType, EnumType, NamedType, UnionType + API_SCHEMA_VERSION = 1 @@ -213,7 +211,7 @@ def from_json(json: Any) -> Class: ClassDocumentation( description=json.get("description", ""), full_docstring=json.get("docstring", ""), - ) + ), ) for method_id in json["methods"]: @@ -229,7 +227,7 @@ def __init__( superclasses: list[str], is_public: bool, reexported_by: list[str], - documentation: ClassDocumentation + documentation: ClassDocumentation, ) -> None: self.id: str = id_ self.qname: str = qname @@ -289,7 +287,7 @@ def from_json(json: Any) -> Function: FunctionDocumentation( description=json.get("description", ""), full_docstring=json.get("docstring", ""), - ) + ), ) @property @@ -319,9 +317,7 @@ def __init__( self.type: Optional[AbstractType] = Type.create_type(parameter_documentation) @classmethod - def create_type( - cls, docstring: ParameterDocumentation - ) -> Optional[AbstractType]: + def create_type(cls, docstring: ParameterDocumentation) -> Optional[AbstractType]: type_string = docstring.type types: list[AbstractType] = list() diff --git a/package-parser/package_parser/processing/dependencies/__init__.py b/package-parser/package_parser/processing/dependencies/__init__.py index 63eda82bd..22deefff2 100644 --- a/package-parser/package_parser/processing/dependencies/__init__.py +++ b/package-parser/package_parser/processing/dependencies/__init__.py @@ -7,14 +7,14 @@ ) from ._parameter_dependencies import ( Action, - RuntimeAction, - StaticAction, Condition, - RuntimeCondition, - StaticCondition, Dependency, ParameterHasValue, ParameterIsIgnored, ParameterIsIllegal, ParameterIsNone, + RuntimeAction, + RuntimeCondition, + StaticAction, + StaticCondition, ) diff --git a/package-parser/package_parser/processing/dependencies/_get_dependency.py b/package-parser/package_parser/processing/dependencies/_get_dependency.py index 5dd4143c7..84f3a6eea 100644 --- a/package-parser/package_parser/processing/dependencies/_get_dependency.py +++ b/package-parser/package_parser/processing/dependencies/_get_dependency.py @@ -1,10 +1,13 @@ from typing import Dict, List, Tuple, Union import spacy -from package_parser.processing.api.model import ( - API, - Parameter, -) +from package_parser.processing.api.model import API, Parameter +from spacy.matcher import DependencyMatcher +from spacy.tokens import Token +from spacy.tokens.doc import Doc +from spacy.tokens.span import Span + +from ._dependency_patterns import dependency_matcher_patterns from ._parameter_dependencies import ( Action, APIDependencies, @@ -15,12 +18,6 @@ ParameterIsIllegal, ParameterIsNone, ) -from spacy.matcher import DependencyMatcher -from spacy.tokens import Token -from spacy.tokens.doc import Doc -from spacy.tokens.span import Span - -from ._dependency_patterns import dependency_matcher_patterns from ._preprocess_docstring import preprocess_docstring PIPELINE = "en_core_web_sm" diff --git a/package-parser/package_parser/processing/usages/_find_usages.py b/package-parser/package_parser/processing/usages/_find_usages.py index c999c24a8..99184c71e 100644 --- a/package-parser/package_parser/processing/usages/_find_usages.py +++ b/package-parser/package_parser/processing/usages/_find_usages.py @@ -6,9 +6,9 @@ import astroid from astroid.builder import AstroidBuilder +from package_parser.processing.usages.model import UsageCountStore from package_parser.utils import ASTWalker, list_files, parse_python_code -from package_parser.processing.usages.model import UsageCountStore from ._ast_visitor import _UsageFinder diff --git a/package-parser/tests/processing/annotations/test_generate_annotations.py b/package-parser/tests/processing/annotations/test_generate_annotations.py index b00aa188e..065031510 100644 --- a/package-parser/tests/processing/annotations/test_generate_annotations.py +++ b/package-parser/tests/processing/annotations/test_generate_annotations.py @@ -2,9 +2,9 @@ import os import pytest +from package_parser.processing.annotations import generate_annotations from package_parser.processing.api.model import API from package_parser.processing.usages.model import UsageCountStore -from package_parser.processing.annotations import generate_annotations @pytest.mark.parametrize( diff --git a/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py b/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py index 3881e0caf..a23218b9c 100644 --- a/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py +++ b/package-parser/tests/processing/api/documentation/test_APIElementDocumentation.py @@ -1,7 +1,9 @@ import pytest - -from package_parser.processing.api.documentation import ClassDocumentation, FunctionDocumentation, \ - ParameterDocumentation +from package_parser.processing.api.documentation import ( + ClassDocumentation, + FunctionDocumentation, + ParameterDocumentation, +) @pytest.mark.parametrize( @@ -9,10 +11,15 @@ [ ClassDocumentation(), ClassDocumentation(description="foo", full_docstring="foo bar"), - ] + ], ) -def test_dict_conversion_for_class_documentation(class_documentation: ClassDocumentation): - assert ClassDocumentation.from_dict(class_documentation.to_dict()) == class_documentation +def test_dict_conversion_for_class_documentation( + class_documentation: ClassDocumentation, +): + assert ( + ClassDocumentation.from_dict(class_documentation.to_dict()) + == class_documentation + ) @pytest.mark.parametrize( @@ -20,10 +27,15 @@ def test_dict_conversion_for_class_documentation(class_documentation: ClassDocum [ FunctionDocumentation(), FunctionDocumentation(description="foo", full_docstring="foo bar"), - ] + ], ) -def test_dict_conversion_for_function_documentation(function_documentation: FunctionDocumentation): - assert FunctionDocumentation.from_dict(function_documentation.to_dict()) == function_documentation +def test_dict_conversion_for_function_documentation( + function_documentation: FunctionDocumentation, +): + assert ( + FunctionDocumentation.from_dict(function_documentation.to_dict()) + == function_documentation + ) @pytest.mark.parametrize( @@ -31,7 +43,12 @@ def test_dict_conversion_for_function_documentation(function_documentation: Func [ ParameterDocumentation(), ParameterDocumentation(type="int", default_value="1", description="foo bar"), - ] + ], ) -def test_dict_conversion_for_parameter_documentation(parameter_documentation: ParameterDocumentation): - assert ParameterDocumentation.from_dict(parameter_documentation.to_dict()) == parameter_documentation +def test_dict_conversion_for_parameter_documentation( + parameter_documentation: ParameterDocumentation, +): + assert ( + ParameterDocumentation.from_dict(parameter_documentation.to_dict()) + == parameter_documentation + ) diff --git a/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py index d5ea2b5ad..be6cf75cf 100644 --- a/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py +++ b/package-parser/tests/processing/api/documentation/test_NumpyDocParser.py @@ -1,8 +1,11 @@ import astroid import pytest - -from package_parser.processing.api.documentation import NumpyDocParser, ClassDocumentation, ParameterDocumentation, \ - FunctionDocumentation +from package_parser.processing.api.documentation import ( + ClassDocumentation, + FunctionDocumentation, + NumpyDocParser, + ParameterDocumentation, +) @pytest.fixture @@ -24,33 +27,36 @@ def __init__(self): ''' # language=python -class_without_documentation = ''' +class_without_documentation = """ class C: pass -''' +""" @pytest.mark.parametrize( "python_code, expected_class_documentation", [ - - (class_with_documentation, ClassDocumentation( - description="Lorem ipsum.\n\nDolor sit amet.", - full_docstring="Lorem ipsum.\n\nDolor sit amet.")), - (class_without_documentation, ClassDocumentation( - description="", - full_docstring="" - )), + ( + class_with_documentation, + ClassDocumentation( + description="Lorem ipsum.\n\nDolor sit amet.", + full_docstring="Lorem ipsum.\n\nDolor sit amet.", + ), + ), + ( + class_without_documentation, + ClassDocumentation(description="", full_docstring=""), + ), ], ids=[ "class with documentation", "class without documentation", - ] + ], ) def test_get_class_documentation( numpydoc_parser: NumpyDocParser, python_code: str, - expected_class_documentation: ClassDocumentation + expected_class_documentation: ClassDocumentation, ): node = astroid.extract_node(python_code) @@ -71,39 +77,44 @@ def f(): ''' # language=python -function_without_documentation = ''' +function_without_documentation = """ def f(): pass -''' +""" @pytest.mark.parametrize( "python_code, expected_function_documentation", [ - - (function_with_documentation, FunctionDocumentation( - description="Lorem ipsum.\n\nDolor sit amet.", - full_docstring="Lorem ipsum.\n\nDolor sit amet.")), - (function_without_documentation, FunctionDocumentation( - description="", - full_docstring="" - )) - + ( + function_with_documentation, + FunctionDocumentation( + description="Lorem ipsum.\n\nDolor sit amet.", + full_docstring="Lorem ipsum.\n\nDolor sit amet.", + ), + ), + ( + function_without_documentation, + FunctionDocumentation(description="", full_docstring=""), + ), ], ids=[ "function with documentation", "function without documentation", - ] + ], ) def test_get_function_documentation( numpydoc_parser: NumpyDocParser, python_code: str, - expected_function_documentation: FunctionDocumentation + expected_function_documentation: FunctionDocumentation, ): node = astroid.extract_node(python_code) assert isinstance(node, astroid.FunctionDef) - assert numpydoc_parser.get_function_documentation(node) == expected_function_documentation + assert ( + numpydoc_parser.get_function_documentation(node) + == expected_function_documentation + ) # language=python @@ -166,62 +177,97 @@ def f( @pytest.mark.parametrize( "python_code, parameter_name, expected_parameter_documentation", [ - - (class_with_parameters, "p", ParameterDocumentation( - type="int", - default_value="1", - description="foo", - )), - (class_with_parameters, "missing", ParameterDocumentation( - type="", - default_value="", - description="", - )), - (function_with_parameters, "no_type_no_default", ParameterDocumentation( - type="", - default_value="", - description="foo: no_type_no_default", - )), - (function_with_parameters, "type_no_default", ParameterDocumentation( - type="int", - default_value="", - description="foo: type_no_default", - )), - (function_with_parameters, "optional_unknown_default", ParameterDocumentation( - type="int", - default_value="", - description="foo: optional_unknown_default", - )), - (function_with_parameters, "with_default_syntax_1", ParameterDocumentation( - type="int", - default_value="1", - description="foo: with_default_syntax_1", - )), - (function_with_parameters, "with_default_syntax_2", ParameterDocumentation( - type="int", - default_value="2", - description="foo: with_default_syntax_2" - )), - (function_with_parameters, "with_default_syntax_3", ParameterDocumentation( - type="int", - default_value="3", - description="foo: with_default_syntax_3" - )), - (function_with_parameters, "grouped_parameter_1", ParameterDocumentation( - type="int", - default_value="4", - description="foo: grouped_parameter_1 and grouped_parameter_2" - )), - (function_with_parameters, "grouped_parameter_2", ParameterDocumentation( - type="int", - default_value="4", - description="foo: grouped_parameter_1 and grouped_parameter_2" - )), - (function_with_parameters, "missing", ParameterDocumentation( - type="", - default_value="", - description="" - )), + ( + class_with_parameters, + "p", + ParameterDocumentation( + type="int", + default_value="1", + description="foo", + ), + ), + ( + class_with_parameters, + "missing", + ParameterDocumentation( + type="", + default_value="", + description="", + ), + ), + ( + function_with_parameters, + "no_type_no_default", + ParameterDocumentation( + type="", + default_value="", + description="foo: no_type_no_default", + ), + ), + ( + function_with_parameters, + "type_no_default", + ParameterDocumentation( + type="int", + default_value="", + description="foo: type_no_default", + ), + ), + ( + function_with_parameters, + "optional_unknown_default", + ParameterDocumentation( + type="int", + default_value="", + description="foo: optional_unknown_default", + ), + ), + ( + function_with_parameters, + "with_default_syntax_1", + ParameterDocumentation( + type="int", + default_value="1", + description="foo: with_default_syntax_1", + ), + ), + ( + function_with_parameters, + "with_default_syntax_2", + ParameterDocumentation( + type="int", default_value="2", description="foo: with_default_syntax_2" + ), + ), + ( + function_with_parameters, + "with_default_syntax_3", + ParameterDocumentation( + type="int", default_value="3", description="foo: with_default_syntax_3" + ), + ), + ( + function_with_parameters, + "grouped_parameter_1", + ParameterDocumentation( + type="int", + default_value="4", + description="foo: grouped_parameter_1 and grouped_parameter_2", + ), + ), + ( + function_with_parameters, + "grouped_parameter_2", + ParameterDocumentation( + type="int", + default_value="4", + description="foo: grouped_parameter_1 and grouped_parameter_2", + ), + ), + ( + function_with_parameters, + "missing", + ParameterDocumentation(type="", default_value="", description=""), + ), ], ids=[ "existing class parameter", @@ -235,13 +281,13 @@ def f( "function parameter with grouped parameters 1", "function parameter with grouped parameters 2", "missing function parameter", - ] + ], ) def test_get_parameter_documentation( numpydoc_parser: NumpyDocParser, python_code: str, parameter_name: str, - expected_parameter_documentation: ParameterDocumentation + expected_parameter_documentation: ParameterDocumentation, ): node = astroid.extract_node(python_code) assert isinstance(node, astroid.ClassDef) or isinstance(node, astroid.FunctionDef) @@ -253,4 +299,7 @@ def test_get_parameter_documentation( node = method assert isinstance(node, astroid.FunctionDef) - assert numpydoc_parser.get_parameter_documentation(node, parameter_name) == expected_parameter_documentation + assert ( + numpydoc_parser.get_parameter_documentation(node, parameter_name) + == expected_parameter_documentation + ) diff --git a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py index b5e749f74..a1fd66d2a 100644 --- a/package-parser/tests/processing/api/documentation/test_get_full_docstring.py +++ b/package-parser/tests/processing/api/documentation/test_get_full_docstring.py @@ -1,6 +1,5 @@ import astroid import pytest - from package_parser.processing.api.documentation import get_full_docstring # language=python @@ -24,10 +23,10 @@ class C: ''' # language=python -class_without_documentation = ''' +class_without_documentation = """ class C: pass -''' +""" # language=python function_with_multi_line_documentation = ''' @@ -50,10 +49,10 @@ def f(): ''' # language=python -function_without_documentation = ''' +function_without_documentation = """ def f(): pass -''' +""" @pytest.mark.parametrize( @@ -73,7 +72,7 @@ def f(): "function with multi line documentation", "function with single line documentation", "function without documentation", - ] + ], ) def test_get_full_docstring(python_code: str, expected_docstring: str): node = astroid.extract_node(python_code) diff --git a/package-parser/tests/processing/api/test_types.py b/package-parser/tests/processing/api/test_types.py index 3a0a6b3bf..da10649dd 100644 --- a/package-parser/tests/processing/api/test_types.py +++ b/package-parser/tests/processing/api/test_types.py @@ -1,7 +1,6 @@ from typing import Any import pytest - from package_parser.processing.api.documentation import ParameterDocumentation from package_parser.processing.api.model import Type @@ -64,7 +63,7 @@ { "kind": "NamedType", "name": "shape (n_samples, n_classes) or (n_samples, 1) when " - "binary.", + "binary.", }, ], }, @@ -137,11 +136,10 @@ def test_boundary_and_union_from_string( docstring_type: str, docstring_description: str, expected: dict[str, Any] ): assert ( - Type(ParameterDocumentation( - type=docstring_type, - default_value="", - description=docstring_description - ) + Type( + ParameterDocumentation( + type=docstring_type, default_value="", description=docstring_description + ) ).to_json() == expected ) diff --git a/package-parser/tests/processing/dependencies/test_get_dependency.py b/package-parser/tests/processing/dependencies/test_get_dependency.py index 155288a39..8db90eb50 100644 --- a/package-parser/tests/processing/dependencies/test_get_dependency.py +++ b/package-parser/tests/processing/dependencies/test_get_dependency.py @@ -1,21 +1,15 @@ import spacy - from package_parser.processing.api.documentation import ParameterDocumentation -from package_parser.processing.api.model import ( - Parameter, - ParameterAssignment -) +from package_parser.processing.api.model import Parameter, ParameterAssignment from package_parser.processing.dependencies import ( Action, Condition, Dependency, + DependencyExtractor, ParameterHasValue, ParameterIsIgnored, ParameterIsIllegal, ParameterIsNone, -) -from package_parser.processing.dependencies import ( - DependencyExtractor, extract_action, extract_condition, extract_lefts_and_rights, @@ -109,7 +103,7 @@ def test_extract_dependencies_from_docstring_pattern_adverbial_clause(): documentation=ParameterDocumentation( type="param possible types", default_value="", - description=param_docstring_nlp.text + description=param_docstring_nlp.text, ), ) dependee_param = Parameter( @@ -122,7 +116,7 @@ def test_extract_dependencies_from_docstring_pattern_adverbial_clause(): documentation=ParameterDocumentation( type="param possible types", default_value="", - description="param probability docstring" + description="param probability docstring", ), ) func_params = [dependent_param, dependee_param] diff --git a/package-parser/tests/processing/usages/model/test_usages.py b/package-parser/tests/processing/usages/model/test_usages.py index be20779c6..703a01d49 100644 --- a/package-parser/tests/processing/usages/model/test_usages.py +++ b/package-parser/tests/processing/usages/model/test_usages.py @@ -1,7 +1,10 @@ from typing import Any import pytest -from package_parser.processing.usages.model import USAGES_SCHEMA_VERSION, UsageCountStore +from package_parser.processing.usages.model import ( + USAGES_SCHEMA_VERSION, + UsageCountStore, +) @pytest.fixture