tests: drop jasmine test runner (#1519)

This patch introduces a tiny test runner to run puppeteer tests.
The test runner is self-container and allows parallel (wrt IO) test execution.
It will also allow us to split tests into multiple files if necessary.

Comparing to the jasmine, the testrunner supports parallel execution, properly
handles "unhandled promise rejection" event and signals.

Comparing to ava/jest, the testrunner doesn't run multiple node processes,
which makes it simpler but sufficient for our goals.

.eslintignore

@@ -1,4 +1,6 @@
 third_party/*
 utils/doclint/check_public_api/test/
+utils/testrunner/examples/
 node6/*
-node6-test/*
+node6-test/*
+node6-testrunner/*

.eslintrc.js

@@ -71,7 +71,7 @@ module.exports = {
         "no-unsafe-negation": 2,
         "radix": 2,
         "valid-typeof": 2,
-        "no-unused-vars": [2, { "args": "none", "vars": "local" }],
+        "no-unused-vars": [2, { "args": "none", "vars": "local", "varsIgnorePattern": "([fx]?describe|[fx]?it|beforeAll|beforeEach|afterAll|afterEach)" }],
         "no-implicit-globals": [2],
 
         // es2015 features

.gitignore

@@ -10,3 +10,4 @@
 package-lock.json
 /node6
 /node6-test
+/node6-testrunner

.travis.yml

@@ -20,7 +20,6 @@ script:
   - 'if [ "$NODE7" = "true" ]; then yarn run lint; fi'
   - 'if [ "$NODE7" = "true" ]; then yarn run coverage; fi'
   - 'if [ "$NODE7" = "true" ]; then yarn run test-doclint; fi'
-  - 'if [ "$NODE6" = "true" ]; then yarn run test-node6-transformer; fi'
   - 'if [ "$NODE6" = "true" ]; then yarn run build; fi'
   - 'if [ "$NODE6" = "true" ]; then yarn run unit-node6; fi'
 jobs:

CONTRIBUTING.md

@@ -111,7 +111,8 @@ A barrier for introducing new installation dependencies is especially high:
 - tests should work on all three platforms: Mac, Linux and Win. This is especially important for screenshot tests.
 
 Puppeteer tests are located in [test/test.js](https://github.com/GoogleChrome/puppeteer/blob/master/test/test.js)
-and are written using [Jasmine](https://jasmine.github.io/) testing framework. Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected.
+and are written with a [TestRunner](https://github.com/GoogleChrome/puppeteer/tree/master/utils/testrunner) framework.
+Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected.
 
 - To run all tests:
 ```

package.json

@@ -8,17 +8,17 @@
     "node": ">=6.4.0"
   },
   "scripts": {
-    "unit": "jasmine test/test.js",
-    "debug-unit": "cross-env DEBUG_TEST=true node --inspect-brk ./node_modules/jasmine/bin/jasmine.js test/test.js",
-    "test-doclint": "jasmine utils/doclint/check_public_api/test/test.js && jasmine utils/doclint/preprocessor/test.js",
+    "unit": "node test/test.js",
+    "debug-unit": "cross-env DEBUG_TEST=true node --inspect-brk test/test.js",
+    "test-doclint": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js",
     "test": "npm run lint --silent && npm run coverage && npm run test-doclint && npm run test-node6-transformer",
     "install": "node install.js",
     "lint": "([ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .) && npm run tsc && npm run doc",
     "doc": "node utils/doclint/cli.js",
     "coverage": "cross-env COVERAGE=true npm run unit",
-    "test-node6-transformer": "jasmine utils/node6-transform/test/test.js",
+    "test-node6-transformer": "node utils/node6-transform/test/test.js",
     "build": "node utils/node6-transform/index.js",
-    "unit-node6": "jasmine node6-test/test.js",
+    "unit-node6": "node node6-test/test.js",
     "tsc": "tsc -p ."
   },
   "author": "The Chromium Authors",
@@ -47,7 +47,6 @@
     "cross-env": "^5.0.5",
     "eslint": "^4.0.0",
     "esprima": "^4.0.0",
-    "jasmine": "^2.6.0",
     "markdown-toc": "^1.1.0",
     "minimist": "^1.2.0",
     "ncp": "^2.0.0",

test/golden-utils.js

@@ -20,15 +20,7 @@ const mime = require('mime');
 const PNG = require('pngjs').PNG;
 const pixelmatch = require('pixelmatch');
 
-module.exports = {
-  addMatchers: function(jasmine, goldenPath, outputPath) {
-    jasmine.addMatchers({
-      toBeGolden: function(util, customEqualityTesters) {
-        return { compare: compare.bind(null, goldenPath, outputPath) };
-      }
-    });
-  },
-};
+module.exports = {compare};
 
 const GoldenComparators = {
   'image/png': compareImages,

test/test.js

@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 const fs = require('fs');
 const rm = require('rimraf').sync;
 const path = require('path');
@@ -26,7 +25,6 @@ const SimpleServer = require('./server/SimpleServer');
 const GoldenUtils = require('./golden-utils');
 
 const YELLOW_COLOR = '\x1b[33m';
-const RED_COLOR = '\x1b[31m';
 const RESET_COLOR = '\x1b[0m';
 
 const GOLDEN_DIR = path.join(__dirname, 'golden');
@@ -52,11 +50,6 @@ const defaultBrowserOptions = {
   args: ['--no-sandbox']
 };
 
-if (process.env.DEBUG_TEST || slowMo)
-  jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 1000 * 1000;
-else
-  jasmine.DEFAULT_TIMEOUT_INTERVAL = 10 * 1000;
-
 // Make sure the `npm install` was run after the chromium roll.
 {
   const Downloader = require('../utils/ChromiumDownloader');
@@ -65,23 +58,18 @@ else
   console.assert(revisionInfo.downloaded, `Chromium r${chromiumRevision} is not downloaded. Run 'npm install' and try to re-run tests.`);
 }
 
-// Hack to get the currently-running spec name.
-let specName = null;
-jasmine.getEnv().addReporter({
-  specStarted: result => specName = result.fullName
-});
+const timeout = process.env.DEBUG_TEST || slowMo ?  0 : 10 * 1000;
+
+const {TestRunner, Reporter, Matchers}  = require('../utils/testrunner/');
+const runner = new TestRunner({timeout});
+new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
 
-// Setup unhandledRejectionHandlers
-let hasUnhandledRejection = false;
-process.on('unhandledRejection', error => {
-  hasUnhandledRejection = true;
-  const textLines = [
-    '',
-    `${RED_COLOR}[UNHANDLED PROMISE REJECTION]${RESET_COLOR} "${specName}"`,
-    error.stack,
-    '',
-  ];
-  console.error(textLines.join('\n'));
+const {expect} = new Matchers({
+  toBeGolden: GoldenUtils.compare.bind(null, GOLDEN_DIR, OUTPUT_DIR)
 });
 
 let server;
@@ -97,7 +85,6 @@ beforeAll(SX(async function() {
 beforeEach(SX(async function() {
   server.reset();
   httpsServer.reset();
-  GoldenUtils.addMatchers(jasmine, GOLDEN_DIR, OUTPUT_DIR);
 }));
 
 afterAll(SX(async function() {
@@ -3280,19 +3267,12 @@ describe('Page', function() {
       serverResponse.end();
       // Wait for the new page to load.
       await waitForEvents(newPage, 'load');
-
-      expect(hasUnhandledRejection).toBe(false);
-
       // Cleanup.
       await newPage.close();
     }));
   });
 });
 
-it('Unhandled promise rejections should not be thrown', function() {
-  expect(hasUnhandledRejection).toBe(false);
-});
-
 if (process.env.COVERAGE) {
   describe('COVERAGE', function(){
     const coverage = helper.publicAPICoverage();
@@ -3307,6 +3287,8 @@ if (process.env.COVERAGE) {
     }
   });
 }
+
+runner.run();
 /**
  * @param {!EventEmitter} emitter
  * @param {string} eventName
@@ -3361,8 +3343,7 @@ function cssPixelsToInches(px) {
   return px / 96;
 }
 
-// Since Jasmine doesn't like async functions, they should be wrapped
-// in a SX function.
+// TODO: remove
 function SX(fun) {
-  return done => Promise.resolve(fun()).then(done).catch(done.fail);
+  return fun;
 }

utils/doclint/check_public_api/test/test.js

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-const fs = require('fs');
-const rm = require('rimraf').sync;
 const path = require('path');
 const puppeteer = require('../../../..');
 const checkPublicAPI = require('..');
@@ -24,45 +22,48 @@ const mdBuilder = require('../MDBuilder');
 const jsBuilder = require('../JSBuilder');
 const GoldenUtils = require('../../../../test/golden-utils');
 
-const OUTPUT_DIR = path.join(__dirname, 'output');
-const GOLDEN_DIR = path.join(__dirname, 'golden');
+const {TestRunner, Reporter, Matchers}  = require('../../../testrunner/');
+const runner = new TestRunner();
+const reporter = new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
 
 let browser;
 let page;
-let specName;
-
-jasmine.getEnv().addReporter({
-  specStarted: result => specName = result.description
-});
 
-beforeAll(SX(async function() {
+beforeAll(async function() {
   browser = await puppeteer.launch({args: ['--no-sandbox']});
   page = await browser.newPage();
-  if (fs.existsSync(OUTPUT_DIR))
-    rm(OUTPUT_DIR);
-}));
+});
 
-afterAll(SX(async function() {
+afterAll(async function() {
   await browser.close();
-}));
+});
 
 describe('checkPublicAPI', function() {
-  it('diff-classes', SX(testLint));
-  it('diff-methods', SX(testLint));
-  it('diff-properties', SX(testLint));
-  it('diff-arguments', SX(testLint));
-  it('diff-events', SX(testLint));
-  it('check-duplicates', SX(testLint));
-  it('check-sorting', SX(testLint));
-  it('check-returns', SX(testLint));
-  it('js-builder-common', SX(testJSBuilder));
-  it('js-builder-inheritance', SX(testJSBuilder));
-  it('md-builder-common', SX(testMDBuilder));
+  it('diff-classes', testLint);
+  it('diff-methods', testLint);
+  it('diff-properties', testLint);
+  it('diff-arguments', testLint);
+  it('diff-events', testLint);
+  it('check-duplicates', testLint);
+  it('check-sorting', testLint);
+  it('check-returns', testLint);
+  it('js-builder-common', testJSBuilder);
+  it('js-builder-inheritance', testJSBuilder);
+  it('md-builder-common', testMDBuilder);
 });
 
-async function testLint() {
-  const dirPath = path.join(__dirname, specName);
-  GoldenUtils.addMatchers(jasmine, dirPath, dirPath);
+runner.run();
+
+async function testLint(state, test) {
+  const dirPath = path.join(__dirname, test.name);
+  const {expect} = new Matchers({
+    toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
+  });
+
   const factory = new SourceFactory();
   const mdSources = await factory.readdir(dirPath, '.md');
   const jsSources = await factory.readdir(dirPath, '.js');
@@ -71,18 +72,22 @@ async function testLint() {
   expect(errors.join('\n')).toBeGolden('result.txt');
 }
 
-async function testMDBuilder() {
-  const dirPath = path.join(__dirname, specName);
-  GoldenUtils.addMatchers(jasmine, dirPath, dirPath);
+async function testMDBuilder(state, test) {
+  const dirPath = path.join(__dirname, test.name);
+  const {expect} = new Matchers({
+    toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
+  });
   const factory = new SourceFactory();
   const sources = await factory.readdir(dirPath, '.md');
   const {documentation} = await mdBuilder(page, sources);
   expect(serialize(documentation)).toBeGolden('result.txt');
 }
 
-async function testJSBuilder() {
-  const dirPath = path.join(__dirname, specName);
-  GoldenUtils.addMatchers(jasmine, dirPath, dirPath);
+async function testJSBuilder(state, test) {
+  const dirPath = path.join(__dirname, test.name);
+  const {expect} = new Matchers({
+    toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
+  });
   const factory = new SourceFactory();
   const sources = await factory.readdir(dirPath, '.js');
   const {documentation} = await jsBuilder(sources);
@@ -110,8 +115,3 @@ function serialize(doc) {
   return JSON.stringify(result, null, 2);
 }
 
-// Since Jasmine doesn't like async functions, they should be wrapped
-// in a SX function.
-function SX(fun) {
-  return done => Promise.resolve(fun()).then(done).catch(done.fail);
-}

utils/doclint/preprocessor/test.js

@@ -19,6 +19,15 @@ const SourceFactory = require('../SourceFactory');
 const factory = new SourceFactory();
 const VERSION = require('../../../package.json').version;
 
+const {TestRunner, Reporter, Matchers}  = require('../../testrunner/');
+const runner = new TestRunner();
+new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+const {expect} = new Matchers();
+
 describe('preprocessor', function() {
   it('should throw for unknown command', function() {
     const source = factory.createForTest('doc.md', getCommand('unknownCommand()'));
@@ -54,6 +63,8 @@ describe('preprocessor', function() {
   });
 });
 
+runner.run();
+
 function getCommand(name, body = '') {
   return `<!--gen:${name}-->${body}<!--gen:stop-->`;
 }

utils/node6-transform/index.js

@@ -21,7 +21,7 @@ const transformAsyncFunctions = require('./TransformAsyncFunctions');
 
 copyFolder(path.join(__dirname, '..', '..', 'lib'), path.join(__dirname, '..', '..', 'node6'));
 copyFolder(path.join(__dirname, '..', '..', 'test'), path.join(__dirname, '..', '..', 'node6-test'));
-
+copyFolder(path.join(__dirname, '..', '..', 'utils', 'testrunner'), path.join(__dirname, '..', '..', 'node6-testrunner'));
 
 function copyFolder(source, target) {
   if (fs.existsSync(target))
@@ -35,8 +35,11 @@ function copyFolder(source, target) {
       copyFolder(from, to);
     } else {
       let text = fs.readFileSync(from);
-      if (file.endsWith('.js'))
-        text = transformAsyncFunctions(text.toString()).replace(/require\('\.\.\/lib\//g, `require('../node6/`);
+      if (file.endsWith('.js')) {
+        text = transformAsyncFunctions(text.toString());
+        text = text.replace(/require\('\.\.\/lib\//g, `require('../node6/`);
+        text = text.replace(/require\('\.\.\/utils\/testrunner\//g, `require('../node6-testrunner/`);
+      }
       fs.writeFileSync(to, text);
     }
   });

utils/node6-transform/test/test.js

@@ -15,6 +15,16 @@
  */
 const transformAsyncFunctions = require('../TransformAsyncFunctions');
 
+const {TestRunner, Reporter, Matchers}  = require('../../testrunner/');
+const runner = new TestRunner();
+new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+const {expect} = new Matchers();
+
 describe('TransformAsyncFunctions', function() {
   it('should convert a function expression', function(done) {
     const input = `(async function(){ return 123 })()`;
@@ -76,4 +86,6 @@ describe('TransformAsyncFunctions', function() {
     expect(output instanceof Promise).toBe(true);
     output.then(result => expect(result).toBe(123)).then(done);
   });
-});
+});
+
+runner.run();