Merge pull request #28 from dmoody256/CoverageTesting

Travis coverage testing with python coverage and coveralls.io

.travis.yml

@@ -1,5 +1,6 @@
 dist: trusty
-before_install:
+
+install:
   - sudo apt-get -y install clang gdc docbook-xml xsltproc libxml2-dev libxslt-dev python-pip python-dev fop docbook-xsl-doc-pdf texlive-full biber texmaker build-essential libpcre3-dev autoconf automake libtool bison subversion git
   - sudo pip install lxml
   - sudo wget http://master.dl.sourceforge.net/project/d-apt/files/d-apt.list -O /etc/apt/sources.list.d/d-apt.list
@@ -12,5 +13,68 @@ before_install:
   - tar xzf rel-3.0.12.tar.gz
   - cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd ..
 
-script: 
-  - python runtest.py -a || if [[ $? == 2 ]]; then exit 0; else exit 1; fi
+jobs:
+  include:
+    - &test_job
+      stage: Test
+      script: 
+        - n=0; while [[ $n -lt 10 ]]; do python runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi
+        - echo "src/engine/SCons/JobTests.py" > exclude_jobtest
+        - python runtest.py -a --exclude-list exclude_jobtest || if [[ $? == 2 ]]; then true; else false; fi
+      before_script: skip
+      after_success: skip
+
+    - &coverage_jobs
+      stage: Coverage
+
+      before_script:
+        - sudo pip install coverage
+        - sudo pip install coveralls
+        - echo "import coverage" | sudo tee --append /usr/lib/python2.7/sitecustomize.py
+        - echo "coverage.process_startup()" | sudo tee --append /usr/lib/python2.7/sitecustomize.py 
+
+      script: 
+        - export TOTAL_BUILD_JOBS=8
+        - export COVERAGE_PROCESS_START=$PWD/.coveragerc
+        - echo "[run]" >> .coveragerc
+        - echo "source = $PWD/src" >> .coveragerc
+        - echo "parallel = True" >> .coveragerc
+      # WORKAROUND: attempt to retry JobTests.py if it fails and then continue if it passes, if it fails ten times
+      # then it is a real failure not related to intermittent travis failures
+        - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi
+      # exclude JobTest.py becuase we already ran that
+        - echo "src/engine/SCons/JobTests.py" > exclude_jobtest
+      # also exclude this test since it overides the exit function which doesnt work with coverage 
+      # more info here: https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#
+      # TODO: figure out how to cover test/exitfns.py
+        - echo "test/exitfns.py" >> exclude_jobtest
+        - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests
+        - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true;
+        - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}"
+        - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi
+        - if (( ${start} == 0 )); then start=1; fi
+        - sed -n ${start},${end}p all_tests > build_tests
+        - coverage run --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi
+
+      after_success:
+        - coverage combine
+        - coverage report
+        - coveralls --rcfile=$PWD/.coveragerc
+
+      env: BUILD_JOB_NUM=1
+
+    - <<: *coverage_jobs
+      env: BUILD_JOB_NUM=2
+    - <<: *coverage_jobs
+      env: BUILD_JOB_NUM=3
+    - <<: *coverage_jobs
+      env: BUILD_JOB_NUM=4
+    - <<: *coverage_jobs
+      env: BUILD_JOB_NUM=5
+    - <<: *coverage_jobs
+      env: BUILD_JOB_NUM=6
+    - <<: *coverage_jobs
+      env: BUILD_JOB_NUM=7
+    - <<: *coverage_jobs
+      env: BUILD_JOB_NUM=8
+

runtest.py

@@ -72,6 +72,10 @@
 #                       This is (will be) used for reporting results back
 #                       to a central SCons test monitoring infrastructure.
 #
+#       --exclude-list file 
+#                       list of tests to exclude in the current selection of test
+#                       mostly meant to easily exclude tests from -a option
+#
 # (Note:  There used to be a -v option that specified the SCons
 # version to be tested, when we were installing in a version-specific
 # library directory.  If we ever resurrect that as the default, then
@@ -126,6 +130,7 @@
 suppress_stderr = False
 allow_pipe_files = True
 quit_on_failure = False
+excludelistfile = None
 
 usagestr = """\
 Usage: runtest.py [OPTIONS] [TEST ...]
@@ -173,6 +178,8 @@
   -X                          Test script is executable, don't feed to Python.
   -x --exec SCRIPT            Test SCRIPT.
      --xml file               Save results to file in SCons XML format.
+     --exclude-list FILE      List of tests to exclude in the current selection of test,
+                              mostly meant to easily exclude tests from the -a option
 
 Environment Variables:
 
@@ -224,7 +231,7 @@ def _process_short_opts(self, rargs, values):
                              'quit-on-failure',
                              'short-progress', 'time',
                              'version=', 'exec=',
-                             'verbose='])
+                             'verbose=', 'exclude-list='])
 
 for o, a in opts:
     if o in ['-b', '--baseline']:
@@ -284,7 +291,8 @@ def _process_short_opts(self, rargs, values):
         scons_exec = 1
     elif o in ['-x', '--exec']:
         scons = a
-
+    elif o in ['--exclude-list']:
+        excludelistfile = a
 
 
 # --- setup stdout/stderr ---
@@ -642,7 +650,7 @@ def footer(self, f):
 # ---[ test discovery ]------------------------------------
 
 tests = []
-
+excludetests = []
 unittests = []
 endtests = []
 
@@ -734,7 +742,7 @@ def find_py(directory):
     tests.extend(unittests)
     tests.extend(endtests)
     tests.sort()
-
+    
 if not tests:
     sys.stderr.write(usagestr + """
 runtest.py:  No tests were found.
@@ -743,9 +751,15 @@ def find_py(directory):
 """)
     sys.exit(1)
 
+if excludelistfile:
+    excludetests = open(excludelistfile, 'r').readlines()
+    excludetests = [x for x in excludetests if x[0] != '#']
+    excludetests = [x[:-1] for x in excludetests]
+    excludetests = [x.strip() for x in excludetests]
+    excludetests = [x for x in excludetests if len(x) > 0]
 
 # ---[ test processing ]-----------------------------------
-
+tests = [t for t in tests if t not in excludetests]
 tests = [Test(t) for t in tests]
 
 if list_only:
@@ -927,4 +941,4 @@ def run(self):
 # tab-width:4
 # indent-tabs-mode:nil
 # End:
-# vim: set expandtab tabstop=4 shiftwidth=4:
+# vim: set expandtab tabstop=4 shiftwidth=4: