Merge c677297 into 9fdc0e3

.travis.yml

@@ -15,18 +15,24 @@ env:
   global:
     - TEST_FUNC_PORT=3030
     - TEST_FUNC_HOST=127.0.0.1
-    - SAUCE_CONNECT_TUNNEL_ID=$TRAVIS_JOB_NUMBER
+    - TEST_PARALLEL=true
+    - SAUCE_CONNECT_TUNNEL_ID=TRAVIS_FMD_REQUIREPACK_${TRAVIS_JOB_NUMBER}
     - SAUCE_USERNAME=requirepack
     - secure: "Gg2no6CkOreDPINT4xoJQPV/93Wq7MxZ7fwgATucuze9KYdidJaKFY8nEx3oJshFtPZ6t46JXxlVrR6n9De720R5RefHUBpuSVVzDedXsdYHXyjh6eNj1z88zteUcLp62hGipvAOrXUeC/l8OuCMCjeqIJtICYF4VJF7RlOyc5RnONOID0u62bogaGnwnHqTH/NlBVLjYDvPGrHhX2FB0TNQaRA89AaE69GyaCSFunEigpA48b/ogfHTF6OfHIvAvtLZf1hrQxGrf854snsqi/yi7+3Kz0m6EABINVGNmD29FqPteR5o7ZPquXt8A8IicL8omFM0jxgU4ma0hRFPvzZLaApFpsCtKfkN9h75LvFkTHkBNXXlaS/ZGMgxWzng6x7jOHxnoczoC2GrqPB48H0kZggHUa7O10gFhmmkbJOywAWv+X2cUaINL/Zn/mxons3JKa1ojb4DDT05TlpUDKv/zPhCAg0fB35GR2/XaZXs1wA8/2fA11VHHfB8Wea91sV+V+3L4H3PD5UueHEV4pE40t54GkjrLVr1OSFIqkDjgVHEkmHJQrx9uVoON6uWT7TwNNK9v2Nt9pRyIYFlHdm1S7pb6h279meN7NC9DueMXp3CYzTwWPgzHGg032tBZ8ga4Nr7wLYZu40Ql+h7D3aw4TWKyPNRo5YM3HkxqRc="
 
 before_install:
   # GUI for real browsers.
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
+  # Upgrade to npm v2
+  - npm install -g npm@2
 
 before_script:
   # Install dev. stuff (e.g., selenium drivers).
   - npm run install-dev
+  # Start up processes used in parallel tests.
+  # Note: We're playing a bit fast & loose here bceause selenium standalone needs to "just be ready"
+  # by the time we start our tests.
   - nohup bash -c "node_modules/.bin/selenium-standalone start 2>&1 &"
   - nohup bash -c "node_modules/.bin/http-server -a $TEST_FUNC_HOST -p $TEST_FUNC_PORT --silent 2>&1 &"
 
@@ -38,8 +44,9 @@ script:
   - npm run check-ci
 
   # Sauce Labs
+  # TODO HERE: --buffer --tries=2 --queue=2
   - >-
-    node test/util/run-concurrent.js "npm run test-func"
+    node_modules/.bin/builder envs test-func --tries=2 --queue=2
     '[ { "ROWDY_SETTINGS":"sauceLabs.IE_8_Windows_2008_Desktop" },
     { "ROWDY_SETTINGS":"sauceLabs.IE_9_Windows_2008_Desktop" },
     { "ROWDY_SETTINGS":"sauceLabs.IE_10_Windows_2012_Desktop" },
@@ -55,4 +62,4 @@ script:
 
   # Manually send coverage reports to coveralls.
   - ls  coverage/*/lcov.info | cat
-  - cat coverage/*/lcov.info | ./node_modules/.bin/coveralls || echo "Coveralls upload failed"
+  - cat coverage/*/lcov.info | node_modules/.bin/coveralls || echo "Coveralls upload failed"

README.md

@@ -48,7 +48,6 @@ Ports various servers run on:
 * **[Tests](docs/test.md)**: Our test suite encompasses a lot of different types
   of builds that are likely to be encountered, running demos, and
 
-
 [webpack]: http://webpack.github.io/
 [wp-dll]: https://github.com/webpack/webpack/tree/master/examples/dll
 [wp-dll-user]: https://github.com/webpack/webpack/tree/master/examples/dll-user

docs/development.md

@@ -1,6 +1,141 @@
 Development
 ===========
 
+We use [builder][] and `npm` to control all aspects of development and
+publishing.
+
+As a preliminary matter, please update your shell to include
+`./node_modules/.bin` in `PATH` like:
+
+```sh
+export PATH="${PATH}:./node_modules/.bin"
+```
+
+So you can type `builder` instead of `./node_modules/.bin/builder` for all
+commands.
+
+
+## Quality
+
+We have various development / CI / coverage helper tasks.
+
+For the overall quality check, run:
+
+```sh
+$ builder run check     # PhantomJS only
+$ builder run check-ci  # (OR) PhantomJS w/ coverage
+```
+
+Which is currently comprised of:
+
+```sh
+$ builder run lint  # AND ...
+
+$ builder run test      # PhantomJS only
+$ builder run test-ci   # (OR) PhantomJS w/ coverage
+```
+
+Note that `(test|check)-(cov|ci)` run code coverage and thus the
+test code may be harder to debug because it is instrumented.
+
+### Parallel Execution
+
+Our CI is setup with a specific optimized parallel workflow. To run parallel
+functional tests in development, here are some helper tasks...
+
+#### Local Browsers
+
+Run parallel functional tests on local browsers:
+
+```sh
+$ TEST_PARALLEL=true \
+  builder envs test-func-local \
+  --setup=setup-local \
+  --buffer \
+  '[ { "ROWDY_SETTINGS":"local.phantomjs" },
+     { "ROWDY_SETTINGS":"local.firefox" },
+     { "ROWDY_SETTINGS":"local.chrome" }
+   ]'
+```
+
+Notes:
+
+* The `TEST_PARALLEL` environment variable signals to _not_ have Rowdy start up
+  selenium and a local dev. server, which is done instead by `setup-local`.
+* The `--buffer` flag stores all parallel output and displays individually as
+  each test task exits for easier comprehensibility.
+* There is a small delay inserted in each task in `test-func-local` to give
+  time for the selenium server to start up.
+
+For a faster, iterative version, use two terminals:
+
+```sh
+# First terminal: Leave persistent tasks running.
+$ builder run setup-local
+
+# Second terminal: Run tests (and repeat)
+$ TEST_PARALLEL=true \
+  builder envs test-func \
+  --buffer \
+  '[ { "ROWDY_SETTINGS":"local.phantomjs" },
+     { "ROWDY_SETTINGS":"local.firefox" },
+     { "ROWDY_SETTINGS":"local.chrome" }
+   ]'
+```
+
+#### Sauce Labs
+
+Run parallel functional tests in Sauce Labs:
+
+```sh
+$ TEST_PARALLEL=true \
+  TEST_FUNC_PORT=3030 \
+  TEST_FUNC_HOST=127.0.0.1 \
+  SAUCE_USERNAME=<INSERT_USERNAME> \
+  SAUCE_ACCESS_KEY=<INSERT_ACCESS_KEY> \
+  builder envs test-func-sauce \
+  --setup=setup-sauce \
+  --buffer \
+  '[ { "ROWDY_SETTINGS":"sauceLabs.IE_8_Windows_2008_Desktop" },
+     { "ROWDY_SETTINGS":"sauceLabs.IE_9_Windows_2008_Desktop" },
+     { "ROWDY_SETTINGS":"sauceLabs.IE_10_Windows_2012_Desktop" }
+   ]'
+```
+
+Notes: This adds an even longer delay in `test-func-sauce` to account for the
+relatively slow startup of Sauce Connect.
+
+For a faster, iterative version, use two terminals:
+
+```sh
+# First terminal: Leave persistent tasks running.
+$ builder run setup-sauce
+
+# Second terminal: Run tests (and repeat)
+$ TEST_PARALLEL=true \
+  TEST_FUNC_PORT=3030 \
+  TEST_FUNC_HOST=127.0.0.1 \
+  SAUCE_USERNAME=<INSERT_USERNAME> \
+  SAUCE_ACCESS_KEY=<INSERT_ACCESS_KEY> \
+  builder envs test-func \
+  --buffer \
+  '[ { "ROWDY_SETTINGS":"sauceLabs.IE_8_Windows_2008_Desktop" },
+     { "ROWDY_SETTINGS":"sauceLabs.IE_9_Windows_2008_Desktop" },
+     { "ROWDY_SETTINGS":"sauceLabs.IE_10_Windows_2012_Desktop" }
+   ]'
+```
+
+### Code Coverage
+
+Code coverage reports are outputted to:
+
+```
+coverage/
+  server|func/
+    lcov-report/index.html  # Viewable web report.
+```
+
+
 ## Releases
 
 **IMPORTANT - NPM**: To correctly run `preversion` your first step is to make
@@ -10,26 +145,21 @@ sure that you have a very modern `npm` binary:
 $ npm install -g npm
 ```
 
-The basic workflow is:
+First, you can optionally edit and commit the project history.
 
 ```sh
-# Make sure you have a clean, up-to-date `master`
-$ git pull
-$ git status # (should be no changes)
-
-# Choose a semantic update for the new version.
-# If you're unsure, read about semantic versioning at http://semver.org/
-$ npm version major|minor|patch -m "Version %s - INSERT_REASONS"
-
-# `package.json` is updated, and files are committed to git (but unpushed).
+$ vim HISTORY.md
+$ git add HISTORY.md
+$ git commit -m "Update history for VERSION"
+```
 
-# Check that everything looks good in last commit and push.
-$ git diff HEAD^ HEAD
-$ git push && git push --tags
-# ... the project is now pushed to GitHub.
+Now we're ready to publish. Choose a semantic update for the new version.
+If you're unsure, read about semantic versioning at http://semver.org/
 
-# And finally publish to `npm`!
-$ npm publish
+```sh
+$ npm version VERSION|major|minor|patch -m "Version %s - INSERT_REASONS"
 ```
 
-And you've published!
+Now `postversion` will push to git and publish to NPM.
+
+[builder]: https://github.com/FormidableLabs/builder

package.json

@@ -10,16 +10,23 @@
     "lint-client": "eslint -c .eslintrc-client test/func/*/src",
     "lint-server": "eslint -c .eslintrc-server lib bin *.js test/func/*/*.config*.js test/func/*/build.js test/util",
     "lint-server-test": "eslint -c .eslintrc-server-test test/func/spec test/func/*/spec",
-    "lint": "npm run lint-client && npm run lint-server && npm run lint-server-test",
-    "test-server": "mocha --opts test/server/mocha.opts test/server/spec",
-    "test-server-cov": "istanbul cover --config .istanbul.server.yml _mocha -- --opts test/server/mocha.opts test/server/spec",
+    "lint": "builder concurrent lint-client lint-server lint-server-test",
+    "test-server": "mocha --color --opts test/server/mocha.opts test/server/spec",
+    "test-server-cov": "istanbul cover --config .istanbul.server.yml _mocha -- --color --opts test/server/mocha.opts test/server/spec",
     "test-func-build": "for s in $(ls test/func/*/build.js); do (node $s); done;",
-    "test-func": "mocha --opts test/func/mocha.opts test/func/spec test/func/*/spec",
-    "test-func-cov": "istanbul cover --config .istanbul.func.yml _mocha -- --opts test/func/mocha.opts test/func/spec test/func/*/spec",
-    "test-ci": "npm run test-server-cov && npm run test-func-build && npm run test-func-cov",
-    "test": "npm run test-server && npm run test-func-build && npm run test-func",
-    "check": "npm run lint && npm run test",
-    "check-ci": "npm run lint && npm run test-ci",
+    "test-func": "mocha --color --opts test/func/mocha.opts test/func/spec test/func/*/spec",
+    "test-func-cov": "istanbul cover --config .istanbul.func.yml _mocha -- --color --opts test/func/mocha.opts test/func/spec test/func/*/spec",
+    "test-ci": "builder run test-server-cov && builder run test-func-build && builder run test-func-cov",
+    "test": "builder run test-server && builder run test-func-build && builder run test-func",
+    "check": "builder run lint && builder run test",
+    "check-ci": "builder run lint && builder run test-ci",
+    "setup-local-server": "http-server -p 3030 --silent >/dev/null",
+    "setup-local-selenium": "selenium-standalone start >/dev/null 2>&1",
+    "setup-local": "builder concurrent setup-local-server setup-local-selenium",
+    "test-func-local": "sleep 5 && echo \"Starting ${ROWDY_SETTINGS}\" && builder run test-func",
+    "setup-sauce-connect": "node_modules/sauce-connect-launcher/sc/*/bin/sc",
+    "setup-sauce": "builder concurrent setup-local-server setup-sauce-connect",
+    "test-func-sauce": "sleep 30 && echo \"Starting ${ROWDY_SETTINGS}\" && builder run test-func",
     "server": "http-server -p 3001 .",
     "install-dev": "selenium-standalone install"
   },
@@ -40,6 +47,7 @@
   },
   "devDependencies": {
     "almond": "^0.3.1",
+    "builder": "FormidableLabs/builder#feature-setup-cleanup2",
     "chai": "^3.3.0",
     "coveralls": "^2.11.4",
     "eslint": "^1.5.1",
@@ -59,11 +67,10 @@
     "rowdy": "^0.3.4",
     "sauce-connect-launcher": "^0.13.0",
     "saucelabs": "^1.0.1",
-    "selenium-standalone": "^4.6.3",
+    "selenium-standalone": "4.7.2",
     "server-destroy": "^1.0.1",
     "sinon": "^1.17.1",
     "sinon-chai": "^2.8.0",
-    "tree-kill": "^1.0.0",
     "webdriverio": "^3.2.4",
     "webpack": "^1.12.2"
   }

test/func/setup.js

@@ -4,7 +4,7 @@
  * Test setup for functional tests.
  */
 var chai = require("chai");
-var startSelenium = process.env.TRAVIS !== "true";
+var startSelenium = process.env.TEST_PARALLEL !== "true";
 
 // Enable Rowdy with webdriverio.
 var _ = require("lodash");

test/func/spec/base.spec.js

@@ -9,52 +9,14 @@
 // Set test environment
 process.env.NODE_ENV = process.env.NODE_ENV || "test-func";
 
-// ----------------------------------------------------------------------------
-// Sauce Connect Tunnel
-// ----------------------------------------------------------------------------
-var rowdy = require("rowdy");
-var isSauceLabs = rowdy.config.setting.isSauceLabs;
-
-if (isSauceLabs && process.env.LAUNCH_SAUCE_CONNECT === "true") {
-  var connect = require("sauce-connect-launcher");
-  var connectPs;
-
-  before(function (done) {
-    // SC takes a **long** time.
-    this.timeout(60000);
-
-    connect({
-      username: rowdy.config.setting.host,
-      accessKey: rowdy.config.setting.key,
-      verbose: true
-    }, function (err, ps) {
-      if (err) { return done(err); }
-      // Stash process.
-      connectPs = ps;
-
-      // Patch settings
-      //obj.desiredCapabilities.tunnelIdentifier =
-
-      done();
-    });
-  });
-
-  after(function (done) {
-    if (connectPs) {
-      this.timeout(30000);
-      return connectPs.close(done);
-    }
-
-    done();
-  });
-}
-
 // ----------------------------------------------------------------------------
 // Selenium (Webdriverio/Rowdy) initialization
 // ----------------------------------------------------------------------------
 // **Note** Can stash adapter, but not `adapter.client` because it is a lazy
 // getter that relies on the global `before|beforeEach` setup.
+var rowdy = require("rowdy");
 var adapter = global.adapter;
+var isSauceLabs = rowdy.config.setting.isSauceLabs;
 var ELEM_WAIT = isSauceLabs ? 5000 : 500; // Global wait.
 
 adapter.before();
@@ -92,7 +54,7 @@ before(function () {
 // App server
 // ----------------------------------------------------------------------------
 before(function (done) {
-  if (process.env.TRAVIS === "true") { return done(); }
+  if (process.env.TEST_PARALLEL === "true") { return done(); }
 
   server = httpServer.createServer();
   server.listen(APP_PORT, APP_HOST, done);