Regression Tests: Run only tests associated with updated example dire…

…ctory (pull #1100)

These changes Cut down the number of tests run on each PR.

* Tests are run by example directory, so all tests for all examples in an example directory will always be run at once.
* But if only one example directory is updated, then only the tests for that example directory will run.
* If any code in `examples/js` or `examples/css` is edited, then all tests are run.
* If `test/index.js` or any code in `test/util` is edited, then all tests are run.
* If package.json is updated, then all tests
are run.
* Changes the name format for the test files. All test files must be prepended with the directory of the example they are testing in order to easily filter tests.
* Test report will report bad test file names.
* Changes the format of the output from AVA's format to TAP.

.travis.yml

@@ -31,14 +31,7 @@ jobs:
 
     - stage: Test
       name: AVA Regression Tests
-      before_install:
-      - |
-        if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qE '(test/|examples/)'
-        then
-          echo "Examples files were not updated, not running example regression tests."
-          exit
-        fi
-      script: ava -c 1 test/tests
+      script: scripts/regression-tests.sh
       env: TEST_WAIT_TIME=1000
     - stage: Test
       name: Regression Tests Coverage Report

scripts/regression-tests.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qP '(test/|examples/|package\.json)'
+then
+  echo "Examples files were not updated, not running example regression tests."
+  exit
+fi
+
+TEST_DIRS=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep -oP 'test/tests/\K[\w-]+(?=_)' | uniq)
+EXAMPLE_DIRS=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep -oP 'examples/\K[\w-]+(?=/)' | uniq)
+
+# Only add match args if the example/js or example/css directories or test/index.hs
+# or the test/utils.js directories have not been edited
+
+TEST_INFRA=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep -oP 'test/(util|index)')
+EXAMPLE_INFRA=$(echo "$EXAMPLE_DIRS" | grep -P '^(js|css)$')
+
+ARGS=''
+
+if [ -z $TEST_INFRA ] && [ -z $EXAMPLE_INFRA ]
+then
+  for D in $EXAMPLE_DIRS
+  do
+    ARGS="${ARGS} --match ${D}/*"
+  done
+
+  for F in $TEST_DIRS
+  do
+    ARGS="${ARGS} --match ${F}/*"
+  done
+fi
+
+AVACMD="npm run regression -- -t -c 1 test/tests ${ARGS}"
+echo "$ $AVACMD"
+
+$AVACMD

test/util/report.js

@@ -26,7 +26,7 @@ const ignoreFiles = fs.readFileSync(ignoreExampleFiles)
 
 /**
  * Recursively find all example pages, saves to exampleFiles global
- * obect.
+ * object.
  *
  * @param {String} currentDirPath - root example directory
  */
@@ -234,6 +234,18 @@ for (let example in exampleCoverage) {
   }
 }
 
+
+let badFileNames = [];
+fs.readdirSync(testsPath).forEach(function (testFile) {
+
+  let dirName = testFile.split('_')[0];
+  let dir = path.join(examplePath, dirName);
+
+  if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
+    badFileNames.push([testFile, dir]);
+  }
+});
+
 console.log('\nExamples without regression tests:\n');
 console.log(examplesWithNoTestsReport || 'None found.\n');
 console.log('\nExamples missing regression tests:\n');
@@ -248,7 +260,16 @@ console.log('  ' + examplesMissingSomeTests + ' example pages are missing approx
             missingTestIds + ' out of approximately ' + totalTestIds + ' tests.\n');
 
 if (examplesMissingSomeTests) {
-  console.log('ERROR:\n\n  Please write missing tests for this report to pass.\n');
+  console.log('ERROR - missing tests:\n\n  Please write missing tests for this report to pass.\n');
+  process.exitCode = 1;
+}
 
+if (badFileNames.length) {
+  console.log('ERROR - bad file names:\n\n  Some test files do not follow the correct naming convention. Test file names should begin with the root parent directory of example being tested followed by an underscore (\'_\'). Please correct the following bad test file(s):\n');
+
+  for (let file of badFileNames) {
+    console.log('  ' + file[0]);
+  }
+  console.log('\n');
   process.exitCode = 1;
 }