Parser and tests for all pass and all failure cases.

Refs #9874

Code/Mantid/docs/sphinxext/mantiddoc/doctest.py

@@ -123,6 +123,9 @@
 ALLPASS_TEST_NAMES_RE = re.compile(r"^\s+(\d+) tests in (.+)$")
 NUMBER_PASSED_RE = re.compile(r"^(\d+) items passed all tests:$")
 
+TEST_FAILED_END_RE = re.compile(r"\*\*\*Test Failed\*\*\* (\d+) failures.")
+FAILURE_LOC_RE = re.compile(r'^File "([\w/\.]+)", line (\d+), in (\w+)$')
+
 #-------------------------------------------------------------------------------
 class TestSuiteReport(object):
 
@@ -245,29 +248,27 @@ def __parse(self, results):
         return TestSuiteReport(name="doctests", cases=cases,
                                package="docs")
 
-    def __parse_document(self, text):
+    def __parse_document(self, results):
         """
         Create a list of TestCaseReport object for this document
 
         Args:
-          text (str): String containing doctest output
-                      for document
+          results (list): List of lines of doctest output
+                          for a single document
         Returns:
           list: List of test cases in the document
         """
-        fullname = self.__extract_fullname(text[0])
-        if not text[1].startswith("-"):
+        fullname = self.__extract_fullname(results[0])
+        if not results[1].startswith("-"):
             raise ValueError("Invalid second line of output: '%s'. "\
                              "Expected a title underline."
                              % text[1])
-
-        text = text[2:] # trim off top two lines
-        if text[0].startswith("*"):
-            print "@todo: Do failure cases"
+        results = results[2:] # trim off top two lines
+        nfailed = self.__find_number_failed(results)
+        if nfailed == 0:
+            testcases = self.__parse_success(fullname, results)
         else:
-            # assume all passed
-            testcases = self.__parse_success(fullname, text)
-
+            testcases = self.__parse_failures(fullname, results, nfailed)
         return testcases
 
     def __extract_fullname(self, line):
@@ -282,23 +283,44 @@ def __extract_fullname(self, line):
                              "beginning '%s'" % DOCTEST_DOCUMENT_BEGIN)
         return line.replace(DOCTEST_DOCUMENT_BEGIN, "").strip()
 
-    def __parse_success(self, fullname, result_txt):
+    def __find_number_failed(self, results):
+        """
+        Returns:
+         Number of failures in the document results
+        """
+        # Last line should be an overall summary
+        lastline = results[-1]
+        if lastline.startswith("***") or lastline == "Test passed.":
+            match = TEST_FAILED_END_RE.match(lastline)
+        else:
+            raise ValueError("Unexpected format for last line of document "
+                             "results. Expected overall summary, "
+                             "found '%s'" % lastline)
+        if match: # regex matched
+            nfailed = int(match.group(1))
+        else:
+            nfailed = 0
+        return nfailed
+
+    def __parse_success(self, fullname, results):
         """
         Parse text for success cases for a single document
 
         Args:
           fullname (str): String containing full name of document
-          result_txt (str): String containing doctest output for
-                            document
+          results (line): List containing lines of doctest output for
+                          document
+        Returns:
+         A list of test cases that were parsed from the results
         """
-        match = NUMBER_PASSED_RE.match(result_txt[0])
+        match = NUMBER_PASSED_RE.match(results[0])
         if not match:
             raise ValueError("All passed line incorrect: '%s'"
-                             % result_txt[0])
-        classname = fullname.split("/")[-1] if "/" in fullname else fullname
+                             % results[0])
+        classname = self.__create_classname(fullname)
         nitems = int(match.group(1))
         cases = []
-        for line in result_txt[1:1+nitems]:
+        for line in results[1:1+nitems]:
             match = ALLPASS_TEST_NAMES_RE.match(line)
             if not match:
                 raise ValueError("Unexpected information line in "
@@ -309,4 +331,69 @@ def __parse_success(self, fullname, result_txt):
         #endfor
         return cases
 
+    def __parse_failures(self, fullname, results, nfailed):
+        """
+        Parse text for failures cases for a single document
+
+        Args:
+          fullname (str): String containing full name of document
+          results (line): List containing lines of doctest output for
+                          document
+          nfailed (int): The number of failures expected in the
+                         document
+        Returns:
+         A list of test cases that were parsed from the results
+        """
+        classname = self.__create_classname(fullname)
+
+        cases = []
+        failure_desc = []
+        indoc = False
+        for line in results:
+            if line.startswith("*"):
+                if len(cases) == nfailed:
+                    break
+                if len(failure_desc) > 0:
+                    cases.append(self.__create_failure_report(classname,
+                                                              failure_desc))
+                failure_desc = []
+                indoc = True
+                continue
+
+            if indoc:
+                failure_desc.append(line)
+
+        return cases
+
+    def __create_classname(self, fullname):
+        """
+        Given a fullname, that can include path separators,
+        produce a classname for the document
+
+        Args:
+          fullname (str): Fullname of document (including paths)
+        """
+        return fullname.replace("/", ".")
+
+    def __create_failure_report(self, classname, failure_desc):
+        """
+        Create a TestCaseReport from a description of the failure. The
+        name is retrieved from the end of first line of description.
+
+        Args:
+          classname (str): A string name for the 'class'
+          failure_desrc (list): A list of lines giving describing the failure
+
+        Returns:
+          A new TestCaseReport object
+        """
+        match = FAILURE_LOC_RE.match(failure_desc[0])
+        if not match:
+            raise ValueError("Unexpected failure description format.\n"
+                             "Expected the first line to contain details "
+                             "of the location of the error.\n"
+                             "Found '%s'" % failure_desc[0])
+        name = match.group(3)
+        return TestCaseReport(classname, name, "\n".join(failure_desc))
+
 #-------------------------------------------------------------------------------

Code/Mantid/docs/sphinxext/mantiddoc/tests/test_doctest.py

@@ -58,7 +58,7 @@ def test_report_stores_expected_attributes_about_test(self):
         self.assertEquals(package, report.package)
         self.assertEquals(testcases, report.testcases)
 
-    def test_report_gives_corret_number_test_passed_and_failed(self):
+    def test_report_gives_correct_number_test_passed_and_failed(self):
         report = self.__createDummyReport()
 
         self.assertEquals(1, report.npassed)
@@ -104,6 +104,44 @@ def __createDummyReport(self, empty = False):
 0 failures in cleanup code
 """
 
+ALL_FAIL_EX = \
+"""Document: algorithms/AllFailed
+------------------------------
+**********************************************************************
+File "algorithms/AllFailed.rst", line 127, in Ex2
+Failed example:
+    print "Multi-line failed"
+    print "test"
+Expected:
+    No match
+Got:
+    Multi-line failed
+    test
+**********************************************************************
+File "algorithms/AllFailed.rst", line 111, in Ex1
+Failed example:
+    print "Single line failed test"
+Expected:
+    No match
+Got:
+    Single line failed test
+**********************************************************************
+2 items had failures:
+   1 of   1 in Ex1
+   1 of   1 in Ex2
+2 tests in 2 items.
+0 passed and 2 failed.
+***Test Failed*** 2 failures.
+
+Doctest summary
+===============
+    2 tests
+    2 failures in tests
+    0 failures in setup code
+    0 failures in cleanup code
+"""
+
+
 class DocTestOutputParserTest(unittest.TestCase):
 
     def test_all_passed_gives_expected_results(self):
@@ -120,16 +158,68 @@ def test_all_passed_gives_expected_results(self):
         for idx, case in enumerate(cases):
             self.assertTrue(case.passed)
             self.assertEquals(expected_names[idx], case.name)
-            self.assertEquals("AllPassed", case.classname)
+            self.assertEquals("algorithms.AllPassed", case.classname)
+
+    def test_all_failed_gives_expected_results(self):
+        parser = DocTestOutputParser(ALL_FAIL_EX, isfile = False)
+
+        self.assertTrue(hasattr(parser, "testsuite"))
+        suite = parser.testsuite
+        self.assertEquals("doctests", suite.name)
+        self.assertEquals("docs", suite.package)
+        self.assertEquals(2, suite.ntests)
+
+        cases = suite.testcases
+        expected_names = ["Ex2", "Ex1"]
+        expected_errors = [
+"""File "algorithms/AllFailed.rst", line 127, in Ex2
+Failed example:
+    print "Multi-line failed"
+    print "test"
+Expected:
+    No match
+Got:
+    Multi-line failed
+    test""", # second error
+"""File "algorithms/AllFailed.rst", line 111, in Ex1
+Failed example:
+    print "Single line failed test"
+Expected:
+    No match
+Got:
+    Single line failed test"""
+]
+        # test
+        for idx, case in enumerate(cases):
+            self.assertTrue(case.failed)
+            self.assertEquals(expected_names[idx], case.name)
+            self.assertEquals(expected_errors[idx], case.failure_descr)
+            self.assertEquals("algorithms.AllFailed", case.classname)
 
     #========================= Failure cases ==================================
 
     def test_no_document_start_gives_valueerror(self):
         self.assertRaises(ValueError, DocTestOutputParser,
                           "----------\n 1 items passed", isfile = False)
 
+    def test_no_location_for_test_failure_gives_valueerror(self):
+        fail_ex_noloc = ALL_FAIL_EX.splitlines()
+        #remove the location line
+        fail_ex_noloc.pop(3)
+        fail_ex_noloc = "\n".join(fail_ex_noloc)
+
+        self.assertRaises(ValueError, DocTestOutputParser, fail_ex_noloc,
+                          isfile = False)
+
+    def test_no_overall_summary_for_document_gives_valueerror(self):
+        fail_ex_nosum = ALL_FAIL_EX.splitlines()
+        fail_ex_nosum.pop(26)
+        fail_ex_nosum = "\n".join(fail_ex_nosum)
+
+        self.assertRaises(ValueError, DocTestOutputParser, fail_ex_nosum,
+                          isfile = False)
+
 #------------------------------------------------------------------------------
 
 if __name__ == '__main__':
     unittest.main()
-