Report RST parsing errors and warnings

Make sure docstrings are properly formatted according to RST in order to
ensure it will be parsed correctly. Testimony now reports any parsing
issue found.

Close #131

testimony/__init__.py

@@ -4,6 +4,7 @@
 
 import ast
 import collections
+import copy
 import fnmatch
 import itertools
 import json
@@ -18,6 +19,7 @@
     PRINT_NO_DOC,
     PRINT_NO_MINIMUM_DOC_TC,
     PRINT_REPORT,
+    PRINT_RST_PARSING_ISSUE,
     PRINT_TOTAL_TC,
     SUMMARY_REPORT,
     VALIDATE_DOCSTRING_REPORT,
@@ -97,6 +99,7 @@ def __init__(self, function_def, parent_class=None, testmodule=None):
             self.pkginit_docstring = None
         self.tokens = {}
         self.invalid_tokens = {}
+        self._rst_parser_messages = []
         self.parser = DocstringParser(
             SETTINGS.get('tokens'),
             SETTINGS.get('minimum_tokens'),
@@ -125,9 +128,11 @@ def _parse_docstring(self):
         for docstring in docstrings:
             if docstring and not isinstance(docstring, type(u'')):
                 docstring = docstring.decode('utf-8')
-            tokens, invalid_tokens = self.parser.parse(docstring)
+            tokens, invalid_tokens, rst_messages = self.parser.parse(docstring)
             self.tokens.update(tokens)
             self.invalid_tokens.update(invalid_tokens)
+            if docstring == docstrings[-1]:
+                self._rst_parser_messages = rst_messages
 
         # Always use the first line of docstring as test case name
         if self.tokens.get('test') is None:
@@ -137,18 +142,66 @@ def _parse_docstring(self):
 
     @property
     def has_valid_docstring(self):
+        """Indicate if the docstring has the minimum tokens."""
+        return self.has_minimum_tokens and not self.has_parsing_issues
+
+    @property
+    def has_minimum_tokens(self):
         """Indicate if the docstring has the minimum tokens."""
         return self.parser.validate_tokens(self.tokens)
 
+    @property
+    def has_parsing_issues(self):
+        """Indicate if the docstring has parsing issues."""
+        return len(self._rst_parser_messages) > 0
+
     def to_dict(self):
         """Return tokens invalid-tokens as a dict."""
         return {
             'tokens': self.tokens.copy(),
             'invalid-tokens': self.invalid_tokens.copy(),
+            'rst-parse-messages': copy.copy(self._rst_parser_messages)
         }
 
+    @property
+    def rst_parser_messages(self):
+        """Return a formatted string with the RST parser messages."""
+        if not self.has_parsing_issues:
+            return ''
+
+        output = []
+        output.append('RST parser messages:\n')
+        for message in self._rst_parser_messages:
+            lines = self.docstring.splitlines()
+            line_index = message.line - 1
+            for index in range(len(lines)):
+                if index == line_index:
+                    lines[index] = '> ' + lines[index]
+                else:
+                    lines[index] = '  ' + lines[index]
+
+            output.append(indent(
+                '* ' + message.message + '\n',
+                ' ' * 2
+            ))
+            docstring_slice = slice(
+                0 if line_index - 2 < 0 else line_index - 2,
+                line_index + 2
+            )
+            output.append(
+                indent(
+                    '\n'.join(lines[docstring_slice]),
+                    ' ' * 4
+                )
+            )
+            output.append('\n')
+            return '\n'.join(output)
+
     def __str__(self):
         """String representation for a test and its tokens."""
+        if self.has_parsing_issues:
+            return self.rst_parser_messages
+
         output = []
         for token, value in sorted(self.tokens.items()):
             output.append('{0}:\n{1}\n'.format(
@@ -269,6 +322,7 @@ def validate_docstring_report(testcases):
     invalid_tags_docstring_count = 0
     minimum_docstring_count = 0
     missing_docstring_count = 0
+    rst_parsing_issue_count = 0
     testcase_count = 0
     for path, tests in testcases.items():
         testcase_count += len(tests)
@@ -277,13 +331,19 @@ def validate_docstring_report(testcases):
             if not testcase.docstring:
                 issues.append('Missing docstring.')
                 missing_docstring_count += 1
-            if not testcase.has_valid_docstring:
+            if not testcase.has_minimum_tokens:
                 issues.append(
                     'Docstring should have at least {} token(s)'.format(
                         ', '.join(sorted(testcase.parser.minimum_tokens))
                     )
                 )
                 minimum_docstring_count += 1
+            if testcase.has_parsing_issues:
+                issues.append(
+                    'Docstring has RST parsing issues. {0}'
+                    .format(testcase.rst_parser_messages)
+                )
+                rst_parsing_issue_count += 1
             if testcase.invalid_tokens:
                 issues.append('Unexpected tokens:\n{0}'.format(
                     indent(
@@ -352,6 +412,15 @@ def validate_docstring_report(testcases):
         colored(invalid_tags_docstring_count, color, attrs=['bold']),
         float(invalid_tags_docstring_count)/testcase_count * 100
     ))
+    if rst_parsing_issue_count == 0:
+        color = CLR_GOOD
+    else:
+        color = CLR_ERR
+    print('{}: {} ({:.02f}%)'.format(
+        PRINT_RST_PARSING_ISSUE.strip(),
+        colored(rst_parsing_issue_count, color, attrs=['bold']),
+        float(rst_parsing_issue_count)/testcase_count * 100
+    ))
 
     if len(result) > 0:
         return -1

testimony/constants.py

@@ -38,3 +38,4 @@
 PRINT_NO_DOC = 'Test cases with no docstrings'
 PRINT_NO_MINIMUM_DOC_TC = 'Test cases missing minimal docstrings'
 PRINT_TOTAL_TC = 'Total number of tests'
+PRINT_RST_PARSING_ISSUE = 'Total number of tests with parsing issues'

testimony/parser.py

@@ -1,13 +1,21 @@
 # coding=utf-8
 """Docstring parser utilities for Testimony."""
+from collections import namedtuple
+from xml.etree import ElementTree
+try:
+    from io import StringIO
+except:
+    from StringIO import StringIO
+
 from docutils.core import publish_string
 from docutils.parsers.rst import nodes, roles
 from docutils.readers import standalone
 from docutils.transforms import frontmatter
-from xml.etree import ElementTree
 
 from testimony.constants import DEFAULT_MINIMUM_TOKENS, DEFAULT_TOKENS
 
+RSTParseMessage = namedtuple('RSTParseMessage', 'line level message')
+
 
 class _NoDocInfoReader(standalone.Reader):
     """Reader that does not do the DocInfo transformation.
@@ -50,38 +58,60 @@ def __init__(self, tokens=None, minimum_tokens=None):
             roles.register_generic_role('py:' + role, nodes.raw)
 
     def parse(self, docstring=None):
-        """Parse the docstring and return the valid and invalid tokens.
+        """Parse docstring and report parsing issues, valid and invalid tokens.
 
         For example in the following docstring (using single quote to demo)::
 
             '''Docstring content.
 
             More docstring content.
 
-            @valid_tag1: value1
-            @valid_tag2: value2
-            @invalid_tag1: value1
-            @invalid_tag2: value2
+            :valid_tag1: value1
+            :valid_tag2: value2
+            :invalid_tag1: value1
+            :invalid_tag2: value2
             '''
 
         Will return a tuple with the following content::
 
             (
                 {'valid_tag1': 'value1', 'valid_tag2': 'value2'},
                 {'invalid_tag1': 'value1', 'invalid_tag2': 'value2'},
+                [],  # List of RSTParseMessage with any formatting issue found
             )
         """
         if docstring is None:
-            return {}, {}
+            return {}, {}, []
         tokens_dict = {}
         valid_tokens = {}
         invalid_tokens = {}
 
         # Parse the docstring with the docutils RST parser and output the
         # result as XML, this ease the process of getting the tokens
         # information.
+        warning_stream = StringIO()
         docstring_xml = publish_string(
-            docstring, reader=_NoDocInfoReader(), writer_name='xml')
+            docstring,
+            reader=_NoDocInfoReader(),
+            settings_overrides={
+                'embed_stylesheet': False,
+                'input_encoding': 'utf-8',
+                'syntax_highlight': 'short',
+                'warning_stream': warning_stream,
+            },
+            writer_name='xml',
+        )
+
+        rst_parse_messages = []
+        for warning in warning_stream.getvalue().splitlines():
+            warning = warning.split(' ', 2)
+            rst_parse_messages.append(RSTParseMessage(
+                line=int(warning[0].split(':')[1]),
+                level=warning[1].split('/')[0][1:].lower(),
+                message=warning[2],
+            ))
+        warning_stream.close()
+
         root = ElementTree.fromstring(docstring_xml)
         tokens = root.findall('./field_list/field')
         for token in tokens:
@@ -107,7 +137,7 @@ def parse(self, docstring=None):
             else:
                 invalid_tokens[token] = value
 
-        return valid_tokens, invalid_tokens
+        return valid_tokens, invalid_tokens, rst_parse_messages
 
     def validate_tokens(self, tokens):
         """Check if the ``tokens`` is a superset of ``minimum_tokens``."""

tests/sample_output.txt

@@ -169,6 +169,20 @@ Test:
  Login with invalid credentials
 
 
+test_invalid_list_style
+-----------------------
+
+RST parser messages:
+
+  * Enumerated list ends without a blank line; unexpected unindent.
+
+      :Steps:
+          1. Have a RST list on any of the tokens, like steps.
+    >     2. Make sure one of the items on the list goes across multiple
+          lines and the lines are not properly indented.
+
+
+
 tests/sample_pkg/test_sample2.py
 ================================
 
@@ -199,17 +213,17 @@ Test:
 = summary report =
 ==================
 
-Total number of tests:          9
-Test cases with no docstrings:  1 (11.11%)
-Assert:                         7 (77.78%)
-Bz:                             2 (22.22%)
-Feature:                        6 (66.67%)
-Setup:                          8 (88.89%)
-Status:                         3 (33.33%)
-Steps:                          7 (77.78%)
-Tags:                           5 (55.56%)
-Test:                           8 (88.89%)
-Type:                           1 (11.11%)
+Total number of tests:          10
+Test cases with no docstrings:   1 (10.00%)
+Assert:                          8 (80.00%)
+Bz:                              2 (20.00%)
+Feature:                         7 (70.00%)
+Setup:                           9 (90.00%)
+Status:                          3 (30.00%)
+Steps:                           8 (80.00%)
+Tags:                            5 (50.00%)
+Test:                            9 (90.00%)
+Type:                            1 (10.00%)
 
 =============================
 = validate_docstring report =
@@ -239,8 +253,23 @@ test_negative_login_5
 
 * Docstring should have at least assert, feature, test token(s)
 
-Total number of tests: 9
-Total number of invalid docstrings: 3 (33.33%)
-Test cases with no docstrings: 1 (11.11%)
-Test cases missing minimal docstrings: 3 (33.33%)
-Test cases with invalid tags: 1 (11.11%)
+test_invalid_list_style
+-----------------------
+
+* Docstring has RST parsing issues. RST parser messages:
+
+  * Enumerated list ends without a blank line; unexpected unindent.
+
+      :Steps:
+          1. Have a RST list on any of the tokens, like steps.
+    >     2. Make sure one of the items on the list goes across multiple
+          lines and the lines are not properly indented.
+
+
+
+Total number of tests: 10
+Total number of invalid docstrings: 4 (40.00%)
+Test cases with no docstrings: 1 (10.00%)
+Test cases missing minimal docstrings: 3 (30.00%)
+Test cases with invalid tags: 1 (10.00%)
+Total number of tests with parsing issues: 1 (10.00%)

tests/test_sample.py

@@ -142,3 +142,24 @@ def test_negative_login_7(self):
         """
         # Code to perform the test
         pass
+
+
+class RSTFormattingTestCase():
+    """Class to test Testimony RST parsing issues features."""
+
+    def test_invalid_list_style(self):
+        """Check invalid list style parsing issue.
+
+        :Feature: RST Parsing Issues
+
+        :Steps:
+            1. Have a RST list on any of the tokens, like steps.
+            2. Make sure one of the items on the list goes across multiple
+            lines and the lines are not properly indented.
+
+        :Assert:
+            1. Testimony reports RST parsing issue for the list with the
+               indentation issue
+            2. Testimony does not report RST parsing issue on list properly
+               formatted.
+    """