Fix subtests (#64)

* experimental - adding coverage button

* Does not crash when tests include subtests

* Output all subtest errors

* Add tests for error and status results

cricket/executor.py

@@ -24,6 +24,29 @@ def enqueue_output(out, queue):
     out.close()
 
 
+def parse_status_and_error(post):
+    if post['status'] == 'OK':
+        status = TestMethod.STATUS_PASS
+        error = None
+    elif post['status'] == 's':
+        status = TestMethod.STATUS_SKIP
+        error = 'Skipped: ' + post.get('error')
+    elif post['status'] == 'F':
+        status = TestMethod.STATUS_FAIL
+        error = post.get('error')
+    elif post['status'] == 'x':
+        status = TestMethod.STATUS_EXPECTED_FAIL
+        error = post.get('error')
+    elif post['status'] == 'u':
+        status = TestMethod.STATUS_UNEXPECTED_SUCCESS
+        error = None
+    elif post['status'] == 'E':
+        status = TestMethod.STATUS_ERROR
+        error = post.get('error')
+
+    return status, error
+
+
 class Executor(EventSource):
     "A wrapper around the subprocess that executes tests."
     def __init__(self, project, count, labels):
@@ -132,26 +155,22 @@ def poll(self):
                     # Start of new test result; record the last result
                     # Then, work out what content goes where.
                     pre = json.loads(self.buffer[0])
-                    post = json.loads(self.buffer[1])
-
-                    if post['status'] == 'OK':
-                        status = TestMethod.STATUS_PASS
-                        error = None
-                    elif post['status'] == 's':
-                        status = TestMethod.STATUS_SKIP
-                        error = 'Skipped: ' + post.get('error')
-                    elif post['status'] == 'F':
-                        status = TestMethod.STATUS_FAIL
-                        error = post.get('error')
-                    elif post['status'] == 'x':
-                        status = TestMethod.STATUS_EXPECTED_FAIL
-                        error = post.get('error')
-                    elif post['status'] == 'u':
-                        status = TestMethod.STATUS_UNEXPECTED_SUCCESS
-                        error = None
-                    elif post['status'] == 'E':
-                        status = TestMethod.STATUS_ERROR
-                        error = post.get('error')
+                    if len(self.buffer) == 2:
+                        # No subtests are present, or only one subtest
+                        post = json.loads(self.buffer[1])
+                        status, error = parse_status_and_error(post)
+
+                    else:
+                        # We have subtests; capture the most important status (until we can capture all the statuses)
+                        status = TestMethod.STATUS_PASS  # Assume pass until told otherwise
+                        error = ''
+                        for line_num in range(1, len(self.buffer)):
+                            post = json.loads(self.buffer[line_num])
+                            subtest_status, subtest_error = parse_status_and_error(post)
+                            if subtest_status > status:
+                                status = subtest_status
+                            if subtest_error:
+                                error += subtest_error + '\n\n'
 
                     # Increase the count of executed tests
                     self.completed_count = self.completed_count + 1

cricket/model.py

@@ -27,10 +27,10 @@ class TestMethod(EventSource):
     """
     STATUS_PASS = 100
     STATUS_SKIP = 200
-    STATUS_FAIL = 300
-    STATUS_EXPECTED_FAIL = 310
-    STATUS_UNEXPECTED_SUCCESS = 320
-    STATUS_ERROR = 400
+    STATUS_EXPECTED_FAIL = 300
+    STATUS_UNEXPECTED_SUCCESS = 400
+    STATUS_FAIL = 500
+    STATUS_ERROR = 600
 
     FAILING_STATES = (STATUS_FAIL, STATUS_UNEXPECTED_SUCCESS, STATUS_ERROR)
 

cricket/pipes.py

@@ -147,6 +147,38 @@ def addFailure(self, test, err):
         self.stream.flush()
         self._current_test = None
 
+    def addSubTest(self, test, subtest, err):
+        super(PipedTestResult, self).addSubTest(test, subtest, err)
+        if err is None:
+            body = {
+                'status': 'OK',
+                'end_time': time.time(),
+                'description': self.description(test),
+                'output': self._stdout.getvalue(),
+            }
+            self.stream.write('%s\n' % json.dumps(body))
+            self.stream.flush()
+        elif issubclass(err[0], test.failureException):
+            body = {
+                'status': 'F',
+                'end_time': time.time(),
+                'description': self.description(test),
+                'error': '\n'.join(traceback.format_exception(*err)),
+                'output': self._stdout.getvalue(),
+            }
+            self.stream.write('%s\n' % json.dumps(body))
+            self.stream.flush()
+        else:
+            body = {
+                'status': 'E',
+                'end_time': time.time(),
+                'description': self.description(test),
+                'error': '\n'.join(traceback.format_exception(*err)),
+                'output': self._stdout.getvalue(),
+            }
+            self.stream.write('%s\n' % json.dumps(body))
+            self.stream.flush()
+
     def addSkip(self, test, reason):
         super(PipedTestResult, self).addSkip(test, reason)
         body = {

cricket/view.py

@@ -208,11 +208,13 @@ def _setup_button_toolbar(self):
         self.rerun_button.grid(column=3, row=0)
 
         self.coverage = StringVar()
+        self.coverage_button = Button(self.toolbar, text='Coverage report', command=self.cmd_show_cov, state=DISABLED)
+        self.coverage_button.grid(column=4, row=0)
         self.coverage_checkbox = Checkbutton(self.toolbar,
                                              text='Generate coverage',
                                              command=self.on_coverageChange,
                                              variable=self.coverage)
-        self.coverage_checkbox.grid(column=4, row=0)
+        self.coverage_checkbox.grid(column=5, row=0)
 
         # If coverage is available, enable it by default.
         # Otherwise, disable the widget
@@ -582,6 +584,9 @@ def cmd_rerun(self, event=None):
         if not self.executor or not self.executor.is_running:
             self.run(status=set(TestMethod.FAILING_STATES))
 
+    def cmd_show_cov(self, event=None):
+        pass
+
     def cmd_open_duvet(self, event=None):
         "Command: Open Duvet"
         try:

tests/test_models.py

@@ -560,7 +560,7 @@ def test_testmethod_collapse(self):
             (3, ['app8.package2.subpackage2.tests2']))
 
     def test_package_collapse(self):
-        "If all test cases in a test pacakge are selected, path is trimmed to the testmethod"
+        "If all test cases in a test package are selected, path is trimmed to the testmethod"
 
         self.assertEquals(self.project.find_tests(labels=[
                 'app6.package2.tests1.TestCase.test_method',

tests/test_utils.py

@@ -0,0 +1,56 @@
+import unittest
+from cricket.executor import parse_status_and_error
+from cricket.model import TestMethod
+
+
+class TestErrorAndStatus(unittest.TestCase):
+    def test_status_returned(self):
+        expected_error_definitions = {
+            'OK': TestMethod.STATUS_PASS,
+            's': TestMethod.STATUS_SKIP,
+            'F': TestMethod.STATUS_FAIL,
+            'x': TestMethod.STATUS_EXPECTED_FAIL,
+            'u': TestMethod.STATUS_UNEXPECTED_SUCCESS,
+            'E': TestMethod.STATUS_ERROR,
+        }
+        for error_code, expected_status in expected_error_definitions.items():
+            with self.subTest('error code ' + error_code):
+                status, error = parse_status_and_error(
+                    {'end_time': 1500000000,
+                     'status': error_code,
+                     'description': 'Some Test',
+                     'error': '',
+                     'output': '',
+                     }
+                )
+                self.assertEqual(status, expected_status)
+
+    def test_error_returned(self):
+        sample_error = 'something broke'
+        expect_error_text = ['s', 'F', 'x', 'E']
+        for error_code in expect_error_text:
+            with self.subTest('error text returned for code ' + error_code):
+                status, error = parse_status_and_error(
+                    {'end_time': 1500000000,
+                     'status': error_code,
+                     'description': 'Some Test',
+                     'error': sample_error,
+                     'output': '',
+                     }
+                )
+                self.assertIn(sample_error, error)
+
+    def test_error_not_returned(self):
+        sample_error = 'something broke'
+        expect_no_error_text = ['OK', 'u']
+        for error_code in expect_no_error_text:
+            with self.subTest('error is none when error code is ' + error_code):
+                status, error = parse_status_and_error(
+                    {'end_time': 1500000000,
+                     'status': error_code,
+                     'description': 'Some Test',
+                     'error': sample_error,
+                     'output': '',
+                     }
+                )
+                self.assertIsNone(error)