Bug 972160 - Run specific marionette tests for 30 times on Travis.

.travis.yml

@@ -16,6 +16,7 @@ env:
   - CI_ACTION=unit-tests-in-b2g
   - CI_ACTION=unit-tests-in-firefox
   - CI_ACTION=marionette_js
+  - CI_ACTION=marionette_js_check
   - CI_ACTION=gaia_ui_tests
   - CI_ACTION=build_tests
   - CI_ACTION=unit-tests-coverage

Makefile

@@ -603,7 +603,6 @@ define run-js-command
 endef
 
 define run-node-command
-	echo "run-node-command $1";
 	node --harmony -e \
 	"require('./build/$(strip $1).js').execute($$BUILD_CONFIG)"
 endef
@@ -713,6 +712,11 @@ test-integration: $(PROFILE_FOLDER)
 		--manifest $(TEST_MANIFEST) \
 		--reporter $(REPORTER)
 
+# Get the file name list of marionete test in the current pull request on Travis.
+.PHONY: pull-request-marionette-test
+pull-request-marionette-test:
+	@$(call run-node-command, pull-request-marionette-test)
+
 .PHONY: test-perf
 test-perf:
 	MOZPERFOUT="$(MOZPERFOUT)" APPS="$(APPS)" MARIONETTE_RUNNER_HOST=$(MARIONETTE_RUNNER_HOST) GAIA_DIR="`pwd`" NPM_REGISTRY=$(NPM_REGISTRY) ./bin/gaia-perf-marionette

build/pull-request-marionette-test.js

@@ -0,0 +1,70 @@
+'use strict';
+
+var request = require('request'),
+    childProcess = require('child_process');
+
+var PULL_REQUEST_URL_PATTERN =
+      'https://api.github.com/repos/mozilla-b2g/gaia/pulls/[id]/files',
+    MARIONETTE_TEST_FILE_NAME_PATTERN =
+      /apps\/[a-z]+\/test\/marionette\/\w+_test.js/;
+
+/**
+ * Get file name list of marionette test in the current pull request on Travis.
+ */
+var PullRequestMarionetteTest = (function() {
+  /**
+   * Get files in a pull request.
+   *
+   * @param {String|Number} id The pull request id.
+   * @param {Function} callback Function with the pull request files array.
+   */
+  function getPullRequestFiles(id, callback) {
+    // Configure the request
+    var options = {
+      url: PULL_REQUEST_URL_PATTERN.replace('[id]', id),
+      method: 'GET',
+      headers: { 'User-Agent': 'FirefoxOS-Gaia-Travis' }
+    };
+
+    request(options, function (error, response, body) {
+      if (!error && response.statusCode == 200) {
+        var json = JSON.parse(body),
+            fileNames = [];
+
+        json.forEach(function(file) {
+          fileNames.push(file.filename);
+        });
+        if (callback && typeof(callback) === 'function') {
+          callback(fileNames);
+        }
+      } else {
+        throw new Error('Cannot access GitHub API.');
+      }
+    });
+  }
+
+  // Main function.
+  function execute() {
+    // Pull request ID for the current Travis job.
+    var pullRequestId = process.env.TRAVIS_PULL_REQUEST;
+
+    getPullRequestFiles(pullRequestId, function(fileNames) {
+      var testFileNames = [],
+          returnString = '';
+
+      testFileNames = fileNames.filter(function getMarionetteTestFileNames(element) {
+                        return element.match(MARIONETTE_TEST_FILE_NAME_PATTERN);
+                      });
+      if (testFileNames.length > 0) {
+        returnString = testFileNames.toString().split(',').join(' ');
+      }
+      console.log(returnString);
+    });
+  }
+
+  return {
+    execute: execute
+  };
+})();
+
+exports.execute = PullRequestMarionetteTest.execute;

package.json

@@ -30,6 +30,7 @@
     "travis-project-jobs": "0.0.1",
     "chai": "~1.4.2",
     "proxyquire": "~0.4",
+    "request": "2.33.0",
     "rimraf": "2.2.5",
     "download": "0.1.7",
     "async": "0.2.9",

tests/travis_ci/marionette_js_check/install

@@ -0,0 +1,7 @@
+#! /bin/bash -ve
+
+echo "Downloading b2g-desktop"
+make b2g
+
+echo "Building test profile"
+PROFILE_FOLDER=profile-test make

tests/travis_ci/marionette_js_check/script

@@ -0,0 +1,19 @@
+#! /bin/bash -v
+
+###############################################################################
+# For stable marionette tests,                                                #
+# we run specfic marionette tests for 30 times on Travis.                     #
+###############################################################################
+
+MARIONETTE_TEST_FILE_NAMES=$(make pull-request-marionette-test)
+
+if [ ${#MARIONETTE_TEST_FILE_NAMES} -gt 0 ]; then
+  RUN_SPECIFIC_TESTS="make test-integration TEST_FILES=\"$MARIONETTE_TEST_FILE_NAMES\" REPORTER=spec" 
+
+  for i in {1..30}
+  do
+    eval ${RUN_SPECIFIC_TESTS}
+  done
+else
+  echo "The pull request has no marionette test to run."
+fi