diff --git a/.eslintignore b/.eslintignore index 8b1421e4b4eb0..e062735bb92bc 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,4 +1,6 @@ third_party/* utils/doclint/check_public_api/test/ +utils/testrunner/examples/ node6/* -node6-test/* \ No newline at end of file +node6-test/* +node6-testrunner/* diff --git a/.eslintrc.js b/.eslintrc.js index 028bb1b158d1a..632a158af6852 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -71,7 +71,7 @@ module.exports = { "no-unsafe-negation": 2, "radix": 2, "valid-typeof": 2, - "no-unused-vars": [2, { "args": "none", "vars": "local" }], + "no-unused-vars": [2, { "args": "none", "vars": "local", "varsIgnorePattern": "([fx]?describe|[fx]?it|beforeAll|beforeEach|afterAll|afterEach)" }], "no-implicit-globals": [2], // es2015 features diff --git a/.gitignore b/.gitignore index e9c7f3851a925..d3a1eb9707a07 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ package-lock.json /node6 /node6-test +/node6-testrunner diff --git a/.travis.yml b/.travis.yml index 2efff4d6633c1..ab72a0a25bac8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,6 @@ script: - 'if [ "$NODE7" = "true" ]; then yarn run lint; fi' - 'if [ "$NODE7" = "true" ]; then yarn run coverage; fi' - 'if [ "$NODE7" = "true" ]; then yarn run test-doclint; fi' - - 'if [ "$NODE6" = "true" ]; then yarn run test-node6-transformer; fi' - 'if [ "$NODE6" = "true" ]; then yarn run build; fi' - 'if [ "$NODE6" = "true" ]; then yarn run unit-node6; fi' jobs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87fd6b9764746..3fd893de0a5b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,7 +111,8 @@ A barrier for introducing new installation dependencies is especially high: - tests should work on all three platforms: Mac, Linux and Win. This is especially important for screenshot tests. Puppeteer tests are located in [test/test.js](https://github.com/GoogleChrome/puppeteer/blob/master/test/test.js) -and are written using [Jasmine](https://jasmine.github.io/) testing framework. Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected. +and are written with a [TestRunner](https://github.com/GoogleChrome/puppeteer/tree/master/utils/testrunner) framework. +Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected. - To run all tests: ``` diff --git a/package.json b/package.json index 6d34b8c3fe121..a1eec7d917a3a 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,17 @@ "node": ">=6.4.0" }, "scripts": { - "unit": "jasmine test/test.js", - "debug-unit": "cross-env DEBUG_TEST=true node --inspect-brk ./node_modules/jasmine/bin/jasmine.js test/test.js", - "test-doclint": "jasmine utils/doclint/check_public_api/test/test.js && jasmine utils/doclint/preprocessor/test.js", + "unit": "node test/test.js", + "debug-unit": "cross-env DEBUG_TEST=true node --inspect-brk test/test.js", + "test-doclint": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js", "test": "npm run lint --silent && npm run coverage && npm run test-doclint && npm run test-node6-transformer", "install": "node install.js", "lint": "([ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .) && npm run tsc && npm run doc", "doc": "node utils/doclint/cli.js", "coverage": "cross-env COVERAGE=true npm run unit", - "test-node6-transformer": "jasmine utils/node6-transform/test/test.js", + "test-node6-transformer": "node utils/node6-transform/test/test.js", "build": "node utils/node6-transform/index.js", - "unit-node6": "jasmine node6-test/test.js", + "unit-node6": "node node6-test/test.js", "tsc": "tsc -p ." }, "author": "The Chromium Authors", @@ -47,7 +47,6 @@ "cross-env": "^5.0.5", "eslint": "^4.0.0", "esprima": "^4.0.0", - "jasmine": "^2.6.0", "markdown-toc": "^1.1.0", "minimist": "^1.2.0", "ncp": "^2.0.0", diff --git a/test/golden-utils.js b/test/golden-utils.js index 096ee526cd7e3..bfa3d266ab8da 100644 --- a/test/golden-utils.js +++ b/test/golden-utils.js @@ -20,15 +20,7 @@ const mime = require('mime'); const PNG = require('pngjs').PNG; const pixelmatch = require('pixelmatch'); -module.exports = { - addMatchers: function(jasmine, goldenPath, outputPath) { - jasmine.addMatchers({ - toBeGolden: function(util, customEqualityTesters) { - return { compare: compare.bind(null, goldenPath, outputPath) }; - } - }); - }, -}; +module.exports = {compare}; const GoldenComparators = { 'image/png': compareImages, diff --git a/test/test.js b/test/test.js index 82545c41e719a..58ca9a8ce301c 100644 --- a/test/test.js +++ b/test/test.js @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - const fs = require('fs'); const rm = require('rimraf').sync; const path = require('path'); @@ -26,7 +25,6 @@ const SimpleServer = require('./server/SimpleServer'); const GoldenUtils = require('./golden-utils'); const YELLOW_COLOR = '\x1b[33m'; -const RED_COLOR = '\x1b[31m'; const RESET_COLOR = '\x1b[0m'; const GOLDEN_DIR = path.join(__dirname, 'golden'); @@ -52,11 +50,6 @@ const defaultBrowserOptions = { args: ['--no-sandbox'] }; -if (process.env.DEBUG_TEST || slowMo) - jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 1000 * 1000; -else - jasmine.DEFAULT_TIMEOUT_INTERVAL = 10 * 1000; - // Make sure the `npm install` was run after the chromium roll. { const Downloader = require('../utils/ChromiumDownloader'); @@ -65,23 +58,18 @@ else console.assert(revisionInfo.downloaded, `Chromium r${chromiumRevision} is not downloaded. Run 'npm install' and try to re-run tests.`); } -// Hack to get the currently-running spec name. -let specName = null; -jasmine.getEnv().addReporter({ - specStarted: result => specName = result.fullName -}); +const timeout = process.env.DEBUG_TEST || slowMo ? 0 : 10 * 1000; + +const {TestRunner, Reporter, Matchers} = require('../utils/testrunner/'); +const runner = new TestRunner({timeout}); +new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; -// Setup unhandledRejectionHandlers -let hasUnhandledRejection = false; -process.on('unhandledRejection', error => { - hasUnhandledRejection = true; - const textLines = [ - '', - `${RED_COLOR}[UNHANDLED PROMISE REJECTION]${RESET_COLOR} "${specName}"`, - error.stack, - '', - ]; - console.error(textLines.join('\n')); +const {expect} = new Matchers({ + toBeGolden: GoldenUtils.compare.bind(null, GOLDEN_DIR, OUTPUT_DIR) }); let server; @@ -97,7 +85,6 @@ beforeAll(SX(async function() { beforeEach(SX(async function() { server.reset(); httpsServer.reset(); - GoldenUtils.addMatchers(jasmine, GOLDEN_DIR, OUTPUT_DIR); })); afterAll(SX(async function() { @@ -3280,19 +3267,12 @@ describe('Page', function() { serverResponse.end(); // Wait for the new page to load. await waitForEvents(newPage, 'load'); - - expect(hasUnhandledRejection).toBe(false); - // Cleanup. await newPage.close(); })); }); }); -it('Unhandled promise rejections should not be thrown', function() { - expect(hasUnhandledRejection).toBe(false); -}); - if (process.env.COVERAGE) { describe('COVERAGE', function(){ const coverage = helper.publicAPICoverage(); @@ -3307,6 +3287,8 @@ if (process.env.COVERAGE) { } }); } + +runner.run(); /** * @param {!EventEmitter} emitter * @param {string} eventName @@ -3361,8 +3343,7 @@ function cssPixelsToInches(px) { return px / 96; } -// Since Jasmine doesn't like async functions, they should be wrapped -// in a SX function. +// TODO: remove function SX(fun) { - return done => Promise.resolve(fun()).then(done).catch(done.fail); + return fun; } diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js index 348e482f2d59f..dd716aeaf4c5d 100644 --- a/utils/doclint/check_public_api/test/test.js +++ b/utils/doclint/check_public_api/test/test.js @@ -14,8 +14,6 @@ * limitations under the License. */ -const fs = require('fs'); -const rm = require('rimraf').sync; const path = require('path'); const puppeteer = require('../../../..'); const checkPublicAPI = require('..'); @@ -24,45 +22,48 @@ const mdBuilder = require('../MDBuilder'); const jsBuilder = require('../JSBuilder'); const GoldenUtils = require('../../../../test/golden-utils'); -const OUTPUT_DIR = path.join(__dirname, 'output'); -const GOLDEN_DIR = path.join(__dirname, 'golden'); +const {TestRunner, Reporter, Matchers} = require('../../../testrunner/'); +const runner = new TestRunner(); +const reporter = new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; let browser; let page; -let specName; - -jasmine.getEnv().addReporter({ - specStarted: result => specName = result.description -}); -beforeAll(SX(async function() { +beforeAll(async function() { browser = await puppeteer.launch({args: ['--no-sandbox']}); page = await browser.newPage(); - if (fs.existsSync(OUTPUT_DIR)) - rm(OUTPUT_DIR); -})); +}); -afterAll(SX(async function() { +afterAll(async function() { await browser.close(); -})); +}); describe('checkPublicAPI', function() { - it('diff-classes', SX(testLint)); - it('diff-methods', SX(testLint)); - it('diff-properties', SX(testLint)); - it('diff-arguments', SX(testLint)); - it('diff-events', SX(testLint)); - it('check-duplicates', SX(testLint)); - it('check-sorting', SX(testLint)); - it('check-returns', SX(testLint)); - it('js-builder-common', SX(testJSBuilder)); - it('js-builder-inheritance', SX(testJSBuilder)); - it('md-builder-common', SX(testMDBuilder)); + it('diff-classes', testLint); + it('diff-methods', testLint); + it('diff-properties', testLint); + it('diff-arguments', testLint); + it('diff-events', testLint); + it('check-duplicates', testLint); + it('check-sorting', testLint); + it('check-returns', testLint); + it('js-builder-common', testJSBuilder); + it('js-builder-inheritance', testJSBuilder); + it('md-builder-common', testMDBuilder); }); -async function testLint() { - const dirPath = path.join(__dirname, specName); - GoldenUtils.addMatchers(jasmine, dirPath, dirPath); +runner.run(); + +async function testLint(state, test) { + const dirPath = path.join(__dirname, test.name); + const {expect} = new Matchers({ + toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) + }); + const factory = new SourceFactory(); const mdSources = await factory.readdir(dirPath, '.md'); const jsSources = await factory.readdir(dirPath, '.js'); @@ -71,18 +72,22 @@ async function testLint() { expect(errors.join('\n')).toBeGolden('result.txt'); } -async function testMDBuilder() { - const dirPath = path.join(__dirname, specName); - GoldenUtils.addMatchers(jasmine, dirPath, dirPath); +async function testMDBuilder(state, test) { + const dirPath = path.join(__dirname, test.name); + const {expect} = new Matchers({ + toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) + }); const factory = new SourceFactory(); const sources = await factory.readdir(dirPath, '.md'); const {documentation} = await mdBuilder(page, sources); expect(serialize(documentation)).toBeGolden('result.txt'); } -async function testJSBuilder() { - const dirPath = path.join(__dirname, specName); - GoldenUtils.addMatchers(jasmine, dirPath, dirPath); +async function testJSBuilder(state, test) { + const dirPath = path.join(__dirname, test.name); + const {expect} = new Matchers({ + toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) + }); const factory = new SourceFactory(); const sources = await factory.readdir(dirPath, '.js'); const {documentation} = await jsBuilder(sources); @@ -110,8 +115,3 @@ function serialize(doc) { return JSON.stringify(result, null, 2); } -// Since Jasmine doesn't like async functions, they should be wrapped -// in a SX function. -function SX(fun) { - return done => Promise.resolve(fun()).then(done).catch(done.fail); -} diff --git a/utils/doclint/preprocessor/test.js b/utils/doclint/preprocessor/test.js index c498203cff154..a260c2439ed63 100644 --- a/utils/doclint/preprocessor/test.js +++ b/utils/doclint/preprocessor/test.js @@ -19,6 +19,15 @@ const SourceFactory = require('../SourceFactory'); const factory = new SourceFactory(); const VERSION = require('../../../package.json').version; +const {TestRunner, Reporter, Matchers} = require('../../testrunner/'); +const runner = new TestRunner(); +new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; +const {expect} = new Matchers(); + describe('preprocessor', function() { it('should throw for unknown command', function() { const source = factory.createForTest('doc.md', getCommand('unknownCommand()')); @@ -54,6 +63,8 @@ describe('preprocessor', function() { }); }); +runner.run(); + function getCommand(name, body = '') { return `${body}`; } diff --git a/utils/node6-transform/index.js b/utils/node6-transform/index.js index 319c457fec4ea..eaf4f9af1c2a8 100644 --- a/utils/node6-transform/index.js +++ b/utils/node6-transform/index.js @@ -21,7 +21,7 @@ const transformAsyncFunctions = require('./TransformAsyncFunctions'); copyFolder(path.join(__dirname, '..', '..', 'lib'), path.join(__dirname, '..', '..', 'node6')); copyFolder(path.join(__dirname, '..', '..', 'test'), path.join(__dirname, '..', '..', 'node6-test')); - +copyFolder(path.join(__dirname, '..', '..', 'utils', 'testrunner'), path.join(__dirname, '..', '..', 'node6-testrunner')); function copyFolder(source, target) { if (fs.existsSync(target)) @@ -35,8 +35,11 @@ function copyFolder(source, target) { copyFolder(from, to); } else { let text = fs.readFileSync(from); - if (file.endsWith('.js')) - text = transformAsyncFunctions(text.toString()).replace(/require\('\.\.\/lib\//g, `require('../node6/`); + if (file.endsWith('.js')) { + text = transformAsyncFunctions(text.toString()); + text = text.replace(/require\('\.\.\/lib\//g, `require('../node6/`); + text = text.replace(/require\('\.\.\/utils\/testrunner\//g, `require('../node6-testrunner/`); + } fs.writeFileSync(to, text); } }); diff --git a/utils/node6-transform/test/test.js b/utils/node6-transform/test/test.js index e14d3b1cb75f7..6c1e0a1ef39dc 100644 --- a/utils/node6-transform/test/test.js +++ b/utils/node6-transform/test/test.js @@ -15,6 +15,16 @@ */ const transformAsyncFunctions = require('../TransformAsyncFunctions'); +const {TestRunner, Reporter, Matchers} = require('../../testrunner/'); +const runner = new TestRunner(); +new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +const {expect} = new Matchers(); + describe('TransformAsyncFunctions', function() { it('should convert a function expression', function(done) { const input = `(async function(){ return 123 })()`; @@ -76,4 +86,6 @@ describe('TransformAsyncFunctions', function() { expect(output instanceof Promise).toBe(true); output.then(result => expect(result).toBe(123)).then(done); }); -}); \ No newline at end of file +}); + +runner.run(); diff --git a/utils/testrunner/Matchers.js b/utils/testrunner/Matchers.js new file mode 100644 index 0000000000000..e3edddff3985e --- /dev/null +++ b/utils/testrunner/Matchers.js @@ -0,0 +1,99 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = class Matchers { + constructor(customMatchers = {}) { + this._matchers = {}; + Object.assign(this._matchers, DefaultMatchers); + Object.assign(this._matchers, customMatchers); + this.expect = this.expect.bind(this); + } + + addMatcher(name, matcher) { + this._matchers[name] = matcher; + } + + expect(value) { + return new Expect(value, this._matchers); + } +}; + +class Expect { + constructor(value, matchers) { + this.not = {}; + this.not.not = this; + for (const matcherName of Object.keys(matchers)) { + const matcher = matchers[matcherName]; + this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false, value); + this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true, value); + } + + function applyMatcher(matcherName, matcher, inverse, value, ...args) { + const result = matcher.call(null, value, ...args); + const message = result.message || `expect.${matcherName} failed`; + console.assert(result.pass !== inverse, message); + } + } +} + +const DefaultMatchers = { + toBe: function(value, other, message) { + return { pass: value === other, message }; + }, + + toBeFalsy: function(value, message) { + return { pass: !value, message }; + }, + + toBeTruthy: function(value, message) { + return { pass: !!value, message }; + }, + + toBeGreaterThan: function(value, other, message) { + return { pass: value > other, message }; + }, + + toBeGreaterThanOrEqual: function(value, other, message) { + return { pass: value >= other, message }; + }, + + toBeLessThan: function(value, other, message) { + return { pass: value < other, message }; + }, + + toBeLessThanOrEqual: function(value, other, message) { + return { pass: value <= other, message }; + }, + + toBeNull: function(value, message) { + return { pass: value === null, message }; + }, + + toContain: function(value, other, message) { + return { pass: value.includes(other), message }; + }, + + toEqual: function(value, other, message) { + return { pass: JSON.stringify(value) === JSON.stringify(other), message }; + }, + + toBeCloseTo: function(value, other, precision, message) { + return { + pass: Math.abs(value - other) < Math.pow(10, -precision), + message + }; + } +}; diff --git a/utils/testrunner/Multimap.js b/utils/testrunner/Multimap.js new file mode 100644 index 0000000000000..6e9c0769a6f2c --- /dev/null +++ b/utils/testrunner/Multimap.js @@ -0,0 +1,95 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class Multimap { + constructor() { + this._map = new Map(); + } + + set(key, value) { + let set = this._map.get(key); + if (!set) { + set = new Set(); + this._map.set(key, set); + } + set.add(value); + } + + get(key) { + let result = this._map.get(key); + if (!result) + result = new Set(); + return result; + } + + has(key) { + return this._map.has(key); + } + + hasValue(key, value) { + const set = this._map.get(key); + if (!set) + return false; + return set.has(value); + } + + /** + * @return {number} + */ + get size() { + return this._map.size; + } + + delete(key, value) { + const values = this.get(key); + const result = values.delete(value); + if (!values.size) + this._map.delete(key); + return result; + } + + deleteAll(key) { + this._map.delete(key); + } + + firstValue(key) { + const set = this._map.get(key); + if (!set) + return null; + return set.values().next().value; + } + + firstKey() { + return this._map.keys().next().value; + } + + valuesArray() { + const result = []; + for (const key of this._map.keys()) + result.push(...Array.from(this._map.get(key).values())); + return result; + } + + keysArray() { + return Array.from(this._map.keys()); + } + + clear() { + this._map.clear(); + } +} + +module.exports = Multimap; diff --git a/utils/testrunner/README.md b/utils/testrunner/README.md new file mode 100644 index 0000000000000..a5887b69d4a8b --- /dev/null +++ b/utils/testrunner/README.md @@ -0,0 +1,45 @@ +# TestRunner + +- no additional binary required; tests are `node.js` scripts +- parallel wrt IO operations +- supports async/await +- modular +- well-isolated state per execution thread + +Example + +```js +const {TestRunner, Reporter, Matchers} = require('../utils/testrunner'); + +// Runner holds and runs all the tests +const runner = new TestRunner({ + parallel: 2, // run 2 parallel threads + timeout: 1000, // setup timeout of 1 second per test +}); +// Simple expect-like matchers +const {expect} = new Matchers(); + +// Extract jasmine-like DSL into the global namespace +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +beforeAll(state => { + state.parallel; // this will be set with the execution thread id, either 0 or 1 in this example + state.foo = 'bar'; // set state for every test +}); + +describe('math', () => { + it('to be sane', async (state, test) => { + state.parallel; // Very first test will always be ran by the 0's thread + state.foo; // this will be 'bar' + expect(2 + 2).toBe(4); + }); +}); + +// Reporter subscribes to TestRunner events and displays information in terminal +const reporter = new Reporter(runner); + +// Run all tests. +runner.run(); +``` diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js new file mode 100644 index 0000000000000..07e6d6c5083d1 --- /dev/null +++ b/utils/testrunner/Reporter.js @@ -0,0 +1,119 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const RED_COLOR = '\x1b[31m'; +const GREEN_COLOR = '\x1b[32m'; +const YELLOW_COLOR = '\x1b[33m'; +const RESET_COLOR = '\x1b[0m'; + +class Reporter { + constructor(runner) { + this._runner = runner; + runner.on('started', this._onStarted.bind(this)); + runner.on('terminated', this._onTerminated.bind(this)); + runner.on('finished', this._onFinished.bind(this)); + runner.on('teststarted', this._onTestStarted.bind(this)); + runner.on('testfinished', this._onTestFinished.bind(this)); + } + + _onStarted() { + this._timestamp = Date.now(); + console.log(`Running ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`); + } + + _onTerminated(message, error) { + this._printTestResults(); + console.log(`${RED_COLOR}## TERMINATED ##${RESET_COLOR}`); + console.log('Message:'); + console.log(` ${RED_COLOR}${message}${RESET_COLOR}`); + if (error && error.stack) { + console.log('Stack:'); + console.log(error.stack.split('\n').map(line => ' ' + line).join('\n')); + } + process.exit(2); + } + + _onFinished() { + this._printTestResults(); + const failedTests = this._runner.failedTests(); + process.exit(failedTests.length > 0 ? 1 : 0); + } + + _printTestResults() { + // 2 newlines after completing all tests. + console.log('\n'); + + const failedTests = this._runner.failedTests(); + if (failedTests.length > 0) { + console.log('\nFailures:'); + for (let i = 0; i < failedTests.length; ++i) { + const test = failedTests[i]; + console.log(`${i + 1}) ${test.fullName}`); + if (test.result === 'timedout') { + console.log(' Message:'); + console.log(` ${YELLOW_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR} ${formatLocation(test)}`); + } else { + console.log(' Message:'); + console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR} ${formatLocation(test)}`); + console.log(' Stack:'); + if (test.error.stack) + console.log(test.error.stack.split('\n').map(line => ' ' + line).join('\n')); + } + console.log(''); + } + } + + const tests = this._runner.tests(); + const skippedTests = tests.filter(test => test.result === 'skipped'); + if (skippedTests.length > 0) { + console.log('\nSkipped:'); + for (let i = 0; i < skippedTests.length; ++i) { + const test = skippedTests[i]; + console.log(`${i + 1}) ${test.fullName}`); + console.log(` ${YELLOW_COLOR}Temporary disabled with xit${RESET_COLOR} ${formatLocation(test)}\n`); + } + } + + const executedTests = tests.filter(test => test.result); + console.log(`\nRan ${executedTests.length} of ${tests.length} test(s)`); + const milliseconds = Date.now() - this._timestamp; + const seconds = milliseconds / 1000; + console.log(`Finished in ${YELLOW_COLOR}${seconds}${RESET_COLOR} seconds`); + + function formatLocation(test) { + const location = test.location; + if (!location) + return ''; + return `@ ${location.fileName}:${location.lineNumber}:${location.columnNumber}`; + } + } + + _onTestStarted() { + } + + _onTestFinished(test) { + if (test.result === 'ok') + process.stdout.write(`${GREEN_COLOR}.${RESET_COLOR}`); + else if (test.result === 'skipped') + process.stdout.write(`${YELLOW_COLOR}*${RESET_COLOR}`); + else if (test.result === 'failed') + process.stdout.write(`${RED_COLOR}F${RESET_COLOR}`); + else if (test.result === 'timedout') + process.stdout.write(`${RED_COLOR}T${RESET_COLOR}`); + } +} + +module.exports = Reporter; diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js new file mode 100644 index 0000000000000..d75817fc1d329 --- /dev/null +++ b/utils/testrunner/TestRunner.js @@ -0,0 +1,381 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); +const EventEmitter = require('events'); +const Multimap = require('./Multimap'); + +const TimeoutError = new Error('Timeout'); +const TerminatedError = new Error('Terminated'); + +class UserCallback { + constructor(callback, timeout) { + this._callback = callback; + this._terminatePromise = new Promise(resolve => { + this._terminateCallback = resolve; + }); + + this.timeout = timeout; + this.location = this._getLocation(); + } + + async run(...args) { + const timeoutPromise = new Promise(resolve => { + if (!this.timeout) + return; + setTimeout(resolve.bind(null, TimeoutError), this.timeout); + }); + try { + return await Promise.race([ + Promise.resolve().then(this._callback.bind(null, ...args)).then(() => null).catch(e => e), + timeoutPromise, + this._terminatePromise + ]); + } catch (e) { + return e; + } + } + + _getLocation() { + const error = new Error(); + const stackFrames = error.stack.split('\n').slice(1); + // Find first stackframe that doesn't point to this file. + for (let frame of stackFrames) { + frame = frame.trim(); + if (!frame.startsWith('at ')) + return null; + if (frame.endsWith(')')) { + const from = frame.indexOf('('); + frame = frame.substring(from + 1, frame.length - 1); + } else { + frame = frame.substring('at '.length + 1); + } + + const match = frame.match(/^(.*):(\d+):(\d+)$/); + if (!match) + return null; + const filePath = match[1]; + const lineNumber = match[2]; + const columnNumber = match[3]; + if (filePath === __filename) + continue; + const fileName = filePath.split(path.sep).pop(); + return { fileName, filePath, lineNumber, columnNumber }; + } + return null; + } + + terminate() { + this._terminateCallback(TerminatedError); + } +} + +const TestMode = { + Run: 'run', + Skip: 'skip', + Focus: 'focus' +}; + +const TestResult = { + Ok: 'ok', + Skipped: 'skipped', // User skipped the test + Failed: 'failed', // Exception happened during running + TimedOut: 'timedout', // Timeout Exceeded while running +}; + +class Test { + constructor(suite, name, callback, declaredMode, timeout) { + this.suite = suite; + this.name = name; + this.fullName = (suite.fullName + ' ' + name).trim(); + this.declaredMode = declaredMode; + this._userCallback = new UserCallback(callback, timeout); + this.location = this._userCallback.location; + + // Test results + this.result = null; + this.error = null; + } +} + +class Suite { + constructor(parentSuite, name, declaredMode) { + this.parentSuite = parentSuite; + this.name = name; + this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim(); + this.declaredMode = declaredMode; + /** @type {!Array<(!Test|!Suite)>} */ + this.children = []; + + this.beforeAll = null; + this.beforeEach = null; + this.afterAll = null; + this.afterEach = null; + } +} + +class TestPass { + constructor(runner, rootSuite, tests, parallel) { + this._runner = runner; + this._parallel = parallel; + this._runningUserCallbacks = new Multimap(); + + this._rootSuite = rootSuite; + this._workerDistribution = new Multimap(); + + let workerId = 0; + for (const test of tests) { + // Reset results for tests that will be run. + test.result = null; + test.error = null; + this._workerDistribution.set(test, workerId); + for (let suite = test.suite; suite; suite = suite.parentSuite) + this._workerDistribution.set(suite, workerId); + // Do not shard skipped tests across workers. + if (test.declaredMode !== TestMode.Skip) + workerId = (workerId + 1) % parallel; + } + + this._termination = null; + } + + async run() { + const terminations = [ + createTermination.call(this, 'SIGINT', 'SIGINT received'), + createTermination.call(this, 'SIGHUP', 'SIGHUP received'), + createTermination.call(this, 'SIGTERM', 'SIGTERM received'), + createTermination.call(this, 'unhandledRejection', 'UNHANDLED PROMISE REJECTION'), + ]; + for (const termination of terminations) + process.on(termination.event, termination.handler); + + const workerPromises = []; + for (let i = 0; i < this._parallel; ++i) + workerPromises.push(this._runSuite(i, [this._rootSuite], {parallel: i})); + await Promise.all(workerPromises); + + for (const termination of terminations) + process.removeListener(termination.event, termination.handler); + return this._termination; + + function createTermination(event, message) { + return { + event, + message, + handler: error => this._terminate(message, error) + }; + } + } + + async _runSuite(workerId, suitesStack, state) { + if (this._termination) + return; + const currentSuite = suitesStack[suitesStack.length - 1]; + if (!this._workerDistribution.hasValue(currentSuite, workerId)) + return; + await this._runHook(workerId, currentSuite, 'beforeAll', state); + for (const child of currentSuite.children) { + if (!this._workerDistribution.hasValue(child, workerId)) + continue; + if (child instanceof Test) { + for (let i = 0; i < suitesStack.length; i++) + await this._runHook(workerId, suitesStack[i], 'beforeEach', state, child); + await this._runTest(workerId, child, state); + for (let i = suitesStack.length - 1; i >= 0; i--) + await this._runHook(workerId, suitesStack[i], 'afterEach', state, child); + } else { + suitesStack.push(child); + await this._runSuite(workerId, suitesStack, state); + suitesStack.pop(); + } + } + await this._runHook(workerId, currentSuite, 'afterAll', state); + } + + async _runTest(workerId, test, state) { + if (this._termination) + return; + this._runner._willStartTest(test); + if (test.declaredMode === TestMode.Skip) { + test.result = TestResult.Skipped; + this._runner._didFinishTest(test); + return; + } + this._runningUserCallbacks.set(workerId, test._userCallback); + const error = await test._userCallback.run(state, test); + this._runningUserCallbacks.delete(workerId, test._userCallback); + if (this._termination) + return; + test.error = error; + if (!error) + test.result = TestResult.Ok; + else if (test.error === TimeoutError) + test.result = TestResult.TimedOut; + else + test.result = TestResult.Failed; + this._runner._didFinishTest(test); + } + + async _runHook(workerId, suite, hookName, ...args) { + if (this._termination) + return; + const hook = suite[hookName]; + if (!hook) + return; + this._runningUserCallbacks.set(workerId, hook); + const error = await hook.run(...args); + this._runningUserCallbacks.delete(workerId, hook); + if (error === TimeoutError) { + const location = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`; + const message = `${location} - Timeout Exceeded ${hook.timeout}ms while running "${hookName}" in suite "${suite.fullName}"`; + this._terminate(message, null); + } else if (error) { + const location = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`; + const message = `${location} - FAILED while running "${hookName}" in suite "${suite.fullName}"`; + this._terminate(message, error); + } + } + + _terminate(message, error) { + if (this._termination) + return; + this._termination = {message, error}; + for (const userCallback of this._runningUserCallbacks.valuesArray()) + userCallback.terminate(); + } +} + +class TestRunner extends EventEmitter { + constructor(options = {}) { + super(); + this._rootSuite = new Suite(null, '', TestMode.Run); + this._currentSuite = this._rootSuite; + this._tests = []; + this._timeout = options.timeout || 10 * 1000; // 10 seconds. + this._parallel = options.parallel || 1; + this._retryFailures = !!options.retryFailures; + + this._hasFocusedTestsOrSuites = false; + + // bind methods so that they can be used as a DSL. + this.describe = this._addSuite.bind(this, TestMode.Run); + this.fdescribe = this._addSuite.bind(this, TestMode.Focus); + this.xdescribe = this._addSuite.bind(this, TestMode.Skip); + this.it = this._addTest.bind(this, TestMode.Run); + this.fit = this._addTest.bind(this, TestMode.Focus); + this.xit = this._addTest.bind(this, TestMode.Skip); + this.beforeAll = this._addHook.bind(this, 'beforeAll'); + this.beforeEach = this._addHook.bind(this, 'beforeEach'); + this.afterAll = this._addHook.bind(this, 'afterAll'); + this.afterEach = this._addHook.bind(this, 'afterEach'); + } + + _addTest(mode, name, callback) { + const test = new Test(this._currentSuite, name, callback, mode, this._timeout); + this._currentSuite.children.push(test); + this._tests.push(test); + this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus; + } + + _addSuite(mode, name, callback) { + const oldSuite = this._currentSuite; + const suite = new Suite(this._currentSuite, name, mode); + this._currentSuite.children.push(suite); + this._currentSuite = suite; + callback(); + this._currentSuite = oldSuite; + this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus; + } + + _addHook(hookName, callback) { + console.assert(this._currentSuite[hookName] === null, `Only one ${hookName} hook available per suite`); + const hook = new UserCallback(callback, this._timeout); + this._currentSuite[hookName] = hook; + } + + async run() { + this.emit(TestRunner.Events.Started); + const pass = new TestPass(this, this._rootSuite, this._runnableTests(), this._parallel); + const termination = await pass.run(); + if (termination) + this.emit(TestRunner.Events.Terminated, termination.message, termination.error); + else + this.emit(TestRunner.Events.Finished); + } + + timeout() { + return this._timeout; + } + + _runnableTests() { + if (!this._hasFocusedTestsOrSuites) + return this._tests; + + const tests = []; + const blacklistSuites = new Set(); + // First pass: pick "fit" and blacklist parent suites + for (const test of this._tests) { + if (test.declaredMode !== TestMode.Focus) + continue; + tests.push(test); + for (let suite = test.suite; suite; suite = suite.parentSuite) + blacklistSuites.add(suite); + } + // Second pass: pick all tests that belong to non-blacklisted "fdescribe" + for (const test of this._tests) { + let insideFocusedSuite = false; + for (let suite = test.suite; suite; suite = suite.parentSuite) { + if (!blacklistSuites.has(suite) && suite.declaredMode === TestMode.Focus) { + insideFocusedSuite = true; + break; + } + } + if (insideFocusedSuite) + tests.push(test); + } + return tests; + } + + tests() { + return this._tests.slice(); + } + + failedTests() { + return this._tests.filter(test => test.result === 'failed' || test.result === 'timedout'); + } + + parallel() { + return this._parallel; + } + + _willStartTest(test) { + this.emit('teststarted', test); + } + + _didFinishTest(test) { + this.emit('testfinished', test); + } +} + +TestRunner.Events = { + Started: 'started', + TestStarted: 'teststarted', + TestFinished: 'testfinished', + Terminated: 'terminated', + Finished: 'finished', +}; + +module.exports = TestRunner; diff --git a/utils/testrunner/examples/fail.js b/utils/testrunner/examples/fail.js new file mode 100644 index 0000000000000..4dddcd7a69fd2 --- /dev/null +++ b/utils/testrunner/examples/fail.js @@ -0,0 +1,33 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter, Matchers} = require('..'); + +const runner = new TestRunner(); +const reporter = new Reporter(runner); +const {expect} = new Matchers(); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + it('failure', async (state) => { + expect(false).toBeTruthy(); + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/hookfail.js b/utils/testrunner/examples/hookfail.js new file mode 100644 index 0000000000000..038f58c47f2fd --- /dev/null +++ b/utils/testrunner/examples/hookfail.js @@ -0,0 +1,35 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter, Matchers} = require('..'); + +const runner = new TestRunner(); +const reporter = new Reporter(runner); +const {expect} = new Matchers(); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + beforeAll(() => { + expect(false).toBeTruthy(); + }); + it('test', async () => { + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/hooktimeout.js b/utils/testrunner/examples/hooktimeout.js new file mode 100644 index 0000000000000..525f2b7c8adf0 --- /dev/null +++ b/utils/testrunner/examples/hooktimeout.js @@ -0,0 +1,35 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter, Matchers} = require('..'); + +const runner = new TestRunner({ timeout: 100 }); +const reporter = new Reporter(runner); +const {expect} = new Matchers(); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + beforeAll(async () => { + await new Promise(() => {}); + }); + it('something', async (state) => { + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/timeout.js b/utils/testrunner/examples/timeout.js new file mode 100644 index 0000000000000..0d451e971e889 --- /dev/null +++ b/utils/testrunner/examples/timeout.js @@ -0,0 +1,32 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter} = require('..'); + +const runner = new TestRunner({ timeout: 100 }); +const reporter = new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + it('timeout', async (state) => { + await new Promise(() => {}); + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/unhandledpromiserejection.js b/utils/testrunner/examples/unhandledpromiserejection.js new file mode 100644 index 0000000000000..270bfcc422171 --- /dev/null +++ b/utils/testrunner/examples/unhandledpromiserejection.js @@ -0,0 +1,35 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter} = require('..'); + +const runner = new TestRunner(); +const reporter = new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + it('failure', async (state) => { + Promise.reject(new Error('fail!')); + }); + it('slow', async () => { + await new Promise(x => setTimeout(x, 1000)); + }); +}); + +runner.run(); diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js new file mode 100644 index 0000000000000..998ea3d150033 --- /dev/null +++ b/utils/testrunner/index.js @@ -0,0 +1,21 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const TestRunner = require('./TestRunner'); +const Reporter = require('./Reporter'); +const Matchers = require('./Matchers'); + +module.exports = { TestRunner, Reporter, Matchers };