fix/test ariatemplates#1681 Using patterns instead of test suites to …

…specify tests to execute

This commit changes test campaigns to use patterns instead of test suites,
in order to remove the need to maintain a list of tests manually.

close ariatemplates#1681

.travis.yml

@@ -5,14 +5,13 @@ sudo: false
 cache:
   directories:
     - node_modules
+addons:
+  firefox: "48.0"
 before_install:
 # Settings needed for graphic output in Firefox
   - export DISPLAY=:99.0
   - /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16
-# This test fails on Travis in Firefox but works locally, TODO investigate that...
-  - sed -i "s/ - test.aria.widgets.container.dialog.movable.test5.MovableDialogTestCaseFive/#/" test/attester-nophantom.yml
-# No Flash on Travis
-  - sed -i "s/ - test.aria.core.io.IOXDRTest/#/" test/attester-nophantom.yml
+  - npm install -g geckodriver
 # Diagnostics
   - npm --version
   - npm config set spin false

package.json

@@ -23,13 +23,9 @@
     "lint-test": "node build/grunt-cli.js checkStyleTest",
     "lint": "node build/grunt-cli.js checkStyle checkStyleTest",
     "grunt": "node build/grunt-cli.js",
-    "attester": "node scripts/attester.js test/attester.yml --env package.json",
-    "attester-nophantom": "sh ./scripts/attester-nophantom.sh",
-    "attester-flatskin": "node scripts/attester.js test/attester-flatskin.yml --env package.json",
-    "attester-packaged": "node scripts/attester.js test/attester-packaged.yml --env package.json",
-    "attester-testskin": "node scripts/attester.js test/attester-testskin.yml --env package.json",
+    "attester": "node scripts/ciCampaigns.js",
     "mocha": "mocha --recursive test/node",
-    "test-suites": "npm run mocha && npm run attester && npm run attester-packaged && npm run attester-flatskin && npm run attester-testskin && npm run attester-nophantom",
+    "test-suites": "npm run mocha && npm run attester",
     "test": "npm run lint && npm run grunt && npm run test-suites",
     "attest": "node scripts/attest.js",
     "ci": "npm run lint-test && npm run test-suites"
@@ -42,14 +38,15 @@
   },
   "devDependencies": {
     "at-diff": "0.0.7",
-    "attester": "2.4.3",
+    "attester": "2.5.0",
     "express": "3.4.8",
     "grunt": "0.4.2",
     "grunt-contrib-jshint": "0.8.0",
     "grunt-leading-indent": "0.1.0",
     "grunt-verifylowercase": "0.2.0",
     "jade": "1.1.5",
     "mocha": "2.3.3",
+    "robot-server": "0.0.4",
     "ua-parser-js": "git+https://github.com/ariatemplates/ua-parser-js#latest"
   }
 }

scripts/attest.js

@@ -14,11 +14,6 @@
  */
 
 var attester = require("attester");
-var middleware = require("./middleware");
-
-middleware.forEach(function (fn) {
-    attester.server.use(fn);
-});
 
 var optionRegExp = /^--([^=]+)=(.*)$/;
 var commandLineOptions = {};
@@ -67,39 +62,9 @@ if (browsers) {
     attesterOptions["robot-browser"] = browser;
 }
 
-function endProcess(code) {
-    attester.dispose().then(function () {
-        process.exit(code);
-    });
-}
-
-// Called when the campaign completes successfully
-attester.event.once("attester.core.idle", function () {
-    endProcess(0);
-});
-
-// Called when the campaign fails
-attester.event.once("attester.core.fail", function () {
-    endProcess(1);
-});
-
 console.log("Using the following options:");
 console.log(JSON.stringify(attesterOptions, null, " "));
 attester.config.set(attesterOptions);
 attester.campaign.create(campaign, {}, 1);
 
-attester.start();
-
-process.on("SIGINT", function() {
-    endProcess(1);
-});
-
-if (process.platform === "win32") {
-    var readline = require("readline").createInterface({
-        input: process.stdin,
-        output: process.stdout
-    });
-    readline.on("SIGINT", function () {
-        process.emit("SIGINT");
-    });
-}
+require("./startAttester");

scripts/ciCampaigns.js

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2014 Amadeus s.a.s.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var attester = require("attester");
+var testConfigBuilder = require("../test/testConfigBuilder");
+var robotServer = require("robot-server");
+var travis = process.env.TRAVIS === "true";
+
+var campaigns = [
+    testConfigBuilder.buildTestConfig({
+        campaign: "classic",
+        browsers: ["PhantomJS"],
+        phantomjs: true
+    }),
+    testConfigBuilder.buildTestConfig({
+        campaign: "unpackaged",
+        browsers: ["PhantomJS"]
+    }),
+    testConfigBuilder.buildTestConfig({
+        campaign: "testSkin",
+        browsers: ["PhantomJS"]
+    }),
+    testConfigBuilder.buildTestConfig({
+        campaign: "flatSkin",
+        browsers: ["PhantomJS"]
+    }),
+    testConfigBuilder.buildTestConfig({
+        campaign: "nophantom",
+        noFlash: travis,
+        browsers: ["Firefox"]
+    })
+];
+
+var robotServerProcess = robotServer.exec();
+process.on("exit", function () {
+    robotServerProcess.kill();
+});
+
+var options = {
+    "colors": true,
+    "env": attester.config.readFile("package.json"),
+    "phantomjs-instances": 0,
+    "launcher-config": "test/ciLauncher.yml"
+};
+attester.config.set(options);
+campaigns.forEach(function (campaign, n) {
+    attester.campaign.create(campaign, {}, n+1);
+});
+
+require("./startAttester");

scripts/startAttester.js

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2016 Amadeus s.a.s.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var attester = require("attester");
+var middleware = require("./middleware");
+
+middleware.forEach(function (fn) {
+    attester.server.use(fn);
+});
+
+function endProcess(code) {
+    attester.dispose().then(function () {
+        process.exit(code);
+    });
+}
+
+// Called when the campaign completes successfully
+attester.event.once("attester.core.idle", function () {
+    endProcess(0);
+});
+
+// Called when the campaign fails
+attester.event.once("attester.core.fail", function () {
+    endProcess(1);
+});
+
+attester.start();
+
+process.on("SIGINT", function() {
+    endProcess(1);
+});
+
+if (process.platform === "win32") {
+    var readline = require("readline").createInterface({
+        input: process.stdin,
+        output: process.stdout
+    });
+    readline.on("SIGINT", function () {
+        process.emit("SIGINT");
+    });
+}

test/ciLauncher.yml

@@ -0,0 +1,15 @@
+launchers:
+    localWebdriver:
+        $launcher: $webdriver
+        $maxInstances: 1
+        $urlExtraParameters: plugin=http://127.0.0.1:7778/robot
+browsers:
+    PhantomJS:
+        $launcher: $phantomjs
+        $maxInstances: 6
+    Firefox:
+        $launcher: localWebdriver
+        capabilities:
+            browserName: firefox
+maxInstances:
+    Launcher_localWebdriver: 1