Merge pull request #4852 from ev-br/ci_build_log

CI build log: try harder to detect test failures

.travis.yml

@@ -73,12 +73,13 @@ before_install:
   - if [ "${TESTMODE}" == "full" ]; then pip install coverage coveralls; fi
   - python -V
   - popd
+  - set -o pipefail
 script:
   - python -c 'import numpy as np; print("relaxed strides checking:", np.ones((10,1),order="C").flags.f_contiguous)'
   # Make sure that relaxed strides checking is actually in effect; otherwise fail loudly
   - if [ "$NPY_RELAXED_STRIDES_CHECKING" = "1" ]; then python -c'import numpy as np; assert np.ones((10,1),order="C").flags.f_contiguous'; fi
   - python -u $OPTIMIZE runtests.py -g -m $TESTMODE $COVERAGE  |& tee runtests.log
-  - python tools/validate_runtests_log.py $TESTMODE < runtests.log
+  - tools/validate_runtests_log.py $TESTMODE < runtests.log
 notifications:
   # Perhaps we should have status emails sent to the mailing list, but
   # let's wait to see what people think before turning that on.

tools/validate_runtests_log.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 """
 Take the test runner log output from the stdin, looking for
 the magic line nose runner prints when the test run was successful.
@@ -35,25 +36,40 @@
     expected_size = {'full': 19055,
                      'fast': 17738}
 
-    # read in the log, parse for the success line.
+    # read in the log, parse for the nose printout:
+    # Ran NNN tests in MMMs
+    # <blank line>
+    # OK (SKIP=X, KNOWNFAIL=Y) or FAILED (errors=X, failures=Y)
     r = re.compile("Ran (?P<num_tests>\d+) tests in (?P<time>\d+\S+)")
 
-    found_it = False
-    for line in sys.stdin:
+    status = False
+    while True:
+        line = sys.stdin.readline()
+        if not line:
+            break
         m = r.search(line)
         if m:
             found_it = True
+            sys.stdin.readline()   # skip the next one
+            line = sys.stdin.readline()
+            if "OK" in line:
+                status = True
             break
 
     if found_it:
+        # did it errored or failed?
+        if not status:
+            print("*** Looks like some tests failed.")
+            sys.exit(-1)
+
         # now check that the number of tests run is reasonable
         expected = expected_size[testmode]
         actual = int(m.group('num_tests'))
         if actual < expected:
-            print("Too few tests: expected %s, run %s" % (expected, actual))
+            print("*** Too few tests: expected %s, run %s" % (expected, actual))
             sys.exit(1)
         else:
             sys.exit(0)
     else:
-        print('Test runner validation errored: did the run really finish?')
+        print('*** Test runner validation errored: did the run really finish?')
         sys.exit(-1)