DocumentationAPI: Add `padding` and `type`

Add `class_padding`, `function_padding` and
`docstring_position` in DocstyleDefinition
which are now acquired from `.coalang`. Add
`top_padding` and `bottom_padding` which is
automatically instantiated from
DocumentationExtraction. Add `docstring_type` in
DocumentationComment which will automatically
determine the type of docstring from
DocumentationExtraction. Add supporting test
cases.

Related to #4200

coalib/bearlib/languages/documentation/DocstyleDefinition.py

@@ -18,10 +18,19 @@ class DocstyleDefinition:
    Metadata = namedtuple('Metadata', ('param_start', 'param_end',
                                       'exception_start', 'exception_end',
                                       'return_sep'))
    ClassPadding = namedtuple('ClassPadding',
                              ('top_padding', 'bottom_padding'))
    FunctionPadding = namedtuple('FunctionPadding',
                                 ('top_padding', 'bottom_padding'))
    DocstringTypeRegex = namedtuple('DocstringTypeRegex',
                                    ('class_sign', 'func_sign'))

    @enforce_signature
    def __init__(self, language: str, docstyle: str, markers: (Iterable, str),
                 metadata: Metadata):
                 metadata: Metadata, class_padding: ClassPadding,
                 function_padding: FunctionPadding,
                 docstring_type_regex: DocstringTypeRegex,
                 docstring_position: str):
        """
        Instantiates a new DocstyleDefinition.

@@ -39,6 +48,18 @@ def __init__(self, language: str, docstyle: str, markers: (Iterable, str),
                         e.g. ``param_start`` defining the start symbol of
                         the parameter fields and ``param_end`` defining the
                         end.
        :param class_padding: A namedtuple consisting of values about
                         blank lines before and after the documentation of
                         ``docstring_type`` class.
        :param function_padding: A namedtuple consisting of values about
                         blank lines before and after the documentation of
                         ``docstring_type`` function.
        :param docstring_type_regex: A namedtuple consisting of regex
                         about ``class`` and ``function`` of a language, which
                         is used to determine ``docstring_type`` of
                         DocumentationComment.
        :param docstring_position: Defines the position where the regex of
                         docstring type is present(i.e. ``top`` or ``bottom``).
        """
        self._language = language.lower()
        self._docstyle = docstyle.lower()
@@ -58,6 +79,10 @@ def __init__(self, language: str, docstyle: str, markers: (Iterable, str),
                                 'actually {}).'.format(length))

        self._metadata = metadata
        self._class_padding = class_padding
        self._function_padding = function_padding
        self._docstring_type_regex = docstring_type_regex
        self._docstring_position = docstring_position

    @property
    def language(self):
@@ -127,6 +152,48 @@ def metadata(self):
        """
        return self._metadata

    @property
    def class_padding(self):
        """
        A namedtuple ``ClassPadding`` consisting of values about blank lines
        before and after the documentation of ``docstring_type`` class.

        These values are official standard of following blank lines before and
        after the documentation of ``docstring_type`` class.
        """
        return self._class_padding

    @property
    def function_padding(self):
        """
        A namedtuple ``FunctionPadding`` consisting of values about blank
        lines before and after the documentation of ``docstring_type``
        function.

        These values are official standard of following blank lines before and
        after the documentation of ``docstring_type`` function.
        """
        return self._function_padding

    @property
    def docstring_type_regex(self):
        """
        A namedtuple ``DocstringTypeRegex`` consisting of regex about ``class``
        and ``function`` of a language, which is used to determine
        ``docstring_type`` of DocumentationComment.
        """
        return self._docstring_type_regex

    @property
    def docstring_position(self):
        """
        Defines the position, where the regex of docstring type is present.
        Depending on different languages the docstrings are present below or
        above the defined class or function. This expicitly defines where the
        class regex or function regex is present(i.e. ``top`` or ``bottom``).
        """
        return self._docstring_position

    @classmethod
    @enforce_signature
    def load(cls, language: str, docstyle: str, coalang_dir=None):
@@ -186,13 +253,44 @@ def load(cls, language: str, docstyle: str, coalang_dir=None):
        metadata = cls.Metadata(*(str(docstyle_settings.get(req_setting, ''))
                                  for req_setting in metadata_settings))

        try:
            class_padding = cls.ClassPadding(
                *(int(padding) for padding in tuple(
                    docstyle_settings['class_padding'])))
        except IndexError:
            class_padding = cls.ClassPadding('', '')

        try:
            function_padding = cls.FunctionPadding(
                *(int(padding) for padding in tuple(
                    docstyle_settings['function_padding'])))
        except IndexError:
            function_padding = cls.FunctionPadding('', '')

        try:
            docstring_type_regex = cls.DocstringTypeRegex(
                *(str(sign) for sign in tuple(
                    docstyle_settings['docstring_type_regex'])))
        except IndexError:
            docstring_type_regex = cls.DocstringTypeRegex('', '')

        try:
            docstring_position = docstyle_settings['docstring_position'].value
        except IndexError:
            docstring_position = ''

        ignore_keys = (('class_padding', 'function_padding',
                        'docstring_type_regex', 'docstring_position') +
                       metadata_settings)

        marker_sets = (tuple(value)
                       for key, value in
                       docstyle_settings.contents.items()
                       if key not in metadata_settings and
                       if key not in ignore_keys and
                       not key.startswith('comment'))

        return cls(language, docstyle, marker_sets, metadata)
        return cls(language, docstyle, marker_sets, metadata, class_padding,
                   function_padding, docstring_type_regex, docstring_position)

    @staticmethod
    def get_available_definitions():

coalib/bearlib/languages/documentation/DocumentationComment.py

@@ -19,6 +19,7 @@ class DocumentationComment:
    Description = namedtuple('Description', 'desc')
    top_padding = 0
    bottom_padding = 0
    docstring_type = 'others'

    def __init__(self, documentation, docstyle_definition,
                 indent, marker, position):

coalib/bearlib/languages/documentation/DocumentationExtraction.py

@@ -3,6 +3,7 @@
from coalib.bearlib.languages.documentation.DocumentationComment import (
    DocumentationComment, MalformedComment)
from coalib.results.TextPosition import TextPosition
from coalib.results.TextRange import TextRange
from textwrap import dedent


@@ -267,5 +268,64 @@ def extract_documentation_with_markers(content, docstyle_definition):
            start_line = doc.range.start.line - 1
            ignore_string_match = ignore_regex.search(content[start_line])

            # Instantiate padding
            top_padding = 0
            bottom_padding = 0
            # minus 2 because we want to check the line before the marker.
            start_index = doc.range.start.line - 2
            end_index = doc.range.end.line
            while start_index >= 0 and not content[start_index].strip():
                top_padding += 1
                start_index -= 1
            # If the end_index is instantiated above the len(content) i.e.
            # In case where ending marker of docstring is at the last line.
            # Then the doc.bottom_padding will be default to 0. This will also
            # prevent IndexError raised by content[end_index].
            while end_index < len(content) and not content[end_index].strip():
                # This condition will take place if theres an inline docstring
                # following documentation.
                if ((doc.marker[2]+'\n') != content[end_index-1][-4:]
                        and bottom_padding == 0):
                    break
                bottom_padding += 1
                end_index += 1

            class_regex = re.compile(
                doc.docstyle_definition.docstring_type_regex.class_sign)
            function_regex = re.compile(
                doc.docstyle_definition.docstring_type_regex.func_sign)

            # End line differs when mid marker and end marker is different
            if doc.marker[1] == doc.marker[2]:
                end_index = end_index - 1

            # Check for docstring_position and then check for class regex
            # and function regex to define the type of docstring.
            if doc.docstyle_definition.docstring_position == 'top':
                if class_regex.search(content[start_index]):
                    doc.docstring_type = 'class'
                elif function_regex.search(content[start_index]):
                    doc.docstring_type = 'function'
            elif doc.docstyle_definition.docstring_position == 'bottom':
                if (end_index < len(content) and
                        class_regex.search(content[end_index])):
                    doc.docstring_type = 'class'
                elif (end_index < len(content) and
                        function_regex.search(content[end_index])):
                    doc.docstring_type = 'function'

            # Disabled automatic padding for docstring_type='others' as this
            # will cause overlapping of range in consecutive docstrings. Which
            # diff.replace() is unable to handle.
            if doc.docstring_type != 'others':
                doc.top_padding = top_padding
                doc.bottom_padding = bottom_padding

                doc.range = TextRange.from_values(
                    start_index + 2,
                    1 if top_padding > 0 else doc.range.start.column,
                    end_index,
                    1 if bottom_padding > 0 else doc.range.end.column)

            if ignore_string_match:
                yield doc

coalib/bearlib/languages/documentation/default.coalang

@@ -5,6 +5,10 @@ param_end = :
exception_start = :raises\ # here's a space
exception_end = :
return_sep = :return:
docstring_type_regex = class, def # defines class and function signature.
docstring_position = top # defines position where docstring regex is found.
class_padding = 0, 1
function_padding = 0, 0

[PYTHON3]
doc-marker = """, , """
@@ -13,6 +17,10 @@ param_end = :
exception_start = :raises\ # here's a space
exception_end = :
return_sep = :return:
docstring_type_regex = class, def # defines class and function signature.
docstring_position = top # defines position where docstring regex is found.
class_padding = 0, 1
function_padding = 0, 0

[JAVA]
doc-marker1 = /**, \ *, \ */

coalib/bearlib/languages/documentation/doxygen.coalang

@@ -39,6 +39,10 @@ param_end = \ # here's a space
exception_start = @raises\ # here's a space
exception_end = \ # here's a space
return_sep = @return\ # here's a space
docstring_type_regex = class, def # defines class and function signature.
docstring_position = bottom # defines position where docstring regex is found.
class_padding = 1, 0
function_padding = 1, 0

[PYTHON3]
doc-marker1 = """, , """
@@ -48,6 +52,10 @@ param_end = \ # here's a space
exception_start = @raises\ # here's a space
exception_end = \ # here's a space
return_sep = @return\ # here's a space
docstring_type_regex = class, def # defines class and function signature.
docstring_position = bottom # defines position where docstring regex is found.
class_padding = 1, 0
function_padding = 1, 0

[TCL]
doc-marker = \#\#, \#, \#