Report tests stuck as inprogress as failures

In some situations (especially when testing compiled extensions) a test
case can trigger a segfault or some other scenario that crashes python.
When this occurs the worker doesn't send back the second subunit event
that reports the test status and that leaves the test case stuck as
inprogress. This becomes tricky to debug as there is no clear message as
to why the test run failed. While we were correctly not returning a 0
return code in this situation there was no error message about why we
didn't. This commit fixes this oversight by adding a check to the end of
subunit-trace so that we report any tests with an inprogress state and
explain that we're failing because they likely crashed python.

stestr/subunit_trace.py

@@ -269,6 +269,18 @@ def count_tests(key, value):
     return count
 
 
+def get_stuck_in_progress():
+    key = 'status'
+    match = re.compile('^inprogress$')
+    in_progress = []
+    for k, v in RESULTS.items():
+        for item in v:
+            if key in item:
+                if match.search(item[key]):
+                    in_progress.append(item['id'])
+    return in_progress
+
+
 def run_time():
     runtime = 0.0
     for k, v in RESULTS.items():
@@ -387,8 +399,12 @@ def trace(stdin, stdout, print_failures=False, failonly=False,
     start_times = []
     stop_times = []
     for worker in RESULTS:
-        start_times += [x['timestamps'][0] for x in RESULTS[worker]]
-        stop_times += [x['timestamps'][1] for x in RESULTS[worker]]
+        start_times += [
+            x['timestamps'][0] for x in RESULTS[worker] if
+            x['timestamps'][0] is not None]
+        stop_times += [
+            x['timestamps'][1] for x in RESULTS[worker] if
+            x['timestamps'][1] is not None]
     start_time = min(start_times)
     stop_time = max(stop_times)
     elapsed_time = stop_time - start_time
@@ -406,6 +422,13 @@ def trace(stdin, stdout, print_failures=False, failonly=False,
     if count_tests('status', '^success$') == 0:
         print("\nNo tests were successful during the run", file=sys.stderr)
         return 1
+    in_progress = get_stuck_in_progress()
+    if count_tests('status', '^inprogress$') > 0:
+        print("\nThe following tests exited without returning a status \n"
+              "and likely segfaulted or crashed Python:", file=sys.stderr)
+        for test in in_progress:
+            print("\n\t* %s" % test, file=sys.stderr)
+        return 1
     return 0 if results.wasSuccessful(summary) else 1
 
 

stestr/tests/test_subunit_trace.py

@@ -22,6 +22,8 @@
 from ddt import data
 from ddt import ddt
 from ddt import unpack
+from subunit.iso8601 import UTC
+from subunit.v2 import StreamResultToBytes
 
 from stestr import subunit_trace
 from stestr.tests import base
@@ -107,3 +109,19 @@ def test_trace_with_failures(self):
         stdin = io.TextIOWrapper(io.BufferedReader(bytes_))
         returncode = subunit_trace.trace(stdin, sys.stdout)
         self.assertEqual(1, returncode)
+
+    def test_trace_with_stuck_inprogress(self):
+        output = io.BytesIO()
+        stream = StreamResultToBytes(output)
+        stream.startTestRun()
+        stream.status(test_id='test_passes', test_status='inprogress',
+                      timestamp=dt.now(UTC))
+        stream.status(test_id='test_segfault', test_status='inprogress',
+                      timestamp=dt.now(UTC))
+        stream.status(test_id='test_passes', test_status='success',
+                      timestamp=dt.now(UTC))
+        stream.stopTestRun()
+        output.seek(0)
+        stdin = io.TextIOWrapper(io.BufferedReader(output))
+        returncode = subunit_trace.trace(stdin, sys.stdout)
+        self.assertEqual(1, returncode)