diff --git a/test-runner/src/fixtures.ts b/test-runner/src/fixtures.ts index 5ad5ab2dba775..a9d9edfa1ace2 100644 --- a/test-runner/src/fixtures.ts +++ b/test-runner/src/fixtures.ts @@ -38,6 +38,19 @@ export function setParameters(params: any) { registerWorkerFixture(name as keyof WorkerState, async ({}, test) => await test(parameters[name] as never)); } +type TestInfo = { + file: string; + title: string; + timeout: number; + outputDir: string; + testDir: string; +}; + +type TestResult = { + success: boolean; + info: TestInfo; + error?: Error; +}; class Fixture { pool: FixturePool; @@ -82,13 +95,13 @@ class Fixture { this._tearDownComplete = this.fn(params, async (value: any) => { this.value = value; setupFenceFulfill(); - await teardownFence; + return await teardownFence; }).catch((e: any) => setupFenceReject(e)); await setupFence; this._setup = true; } - async teardown() { + async teardown(testResult: TestResult) { if (this.hasGeneratorValue) return; if (this._teardown) @@ -98,11 +111,11 @@ class Fixture { const fixture = this.pool.instances.get(name); if (!fixture) continue; - await fixture.teardown(); + await fixture.teardown(testResult); } if (this._setup) { debug('pw:test:hook')(`teardown "${this.name}"`); - this._teardownFenceCallback(); + this._teardownFenceCallback(testResult); } await this._tearDownComplete; this.pool.instances.delete(this.name); @@ -129,31 +142,45 @@ export class FixturePool { return fixture; } - async teardownScope(scope: string) { + async teardownScope(scope: string, testResult?: TestResult) { for (const [name, fixture] of this.instances) { if (fixture.scope === scope) - await fixture.teardown(); + await fixture.teardown(testResult); } } - async resolveParametersAndRun(fn: (arg0: {}) => any) { + async resolveParametersAndRun(fn: (arg0: {}) => any, timeout: number) { const names = fixtureParameterNames(fn); for (const name of names) await this.setupFixture(name); const params = {}; for (const n of names) params[n] = this.instances.get(n).value; - return fn(params); + + if (!timeout) + return fn(params); + + let timer; + let timerPromise = new Promise(f => timer = setTimeout(f, timeout)); + return Promise.race([ + Promise.resolve(fn(params)).then(() => clearTimeout(timer)), + timerPromise.then(() => Promise.reject(new Error(`Timeout of ${timeout}ms exceeded`))) + ]); } - wrapTestCallback(callback: any) { + wrapTestCallback(callback: any, timeout: number, info: TestInfo) { if (!callback) return callback; + const testResult: TestResult = { success: true, info }; return async() => { try { - return await this.resolveParametersAndRun(callback); + await this.resolveParametersAndRun(callback, timeout); + } catch (e) { + testResult.success = false; + testResult.error = e; + throw e; } finally { - await this.teardownScope('test'); + await this.teardownScope('test', testResult); } }; } @@ -188,15 +215,6 @@ function fixtureParameterNames(fn: { toString: () => any; }) { return signature.split(',').map((t: string) => t.trim()); } -function optionParameterNames(fn: { toString: () => any; }) { - const text = fn.toString(); - const match = text.match(/(?:\s+function)?\s*\(\s*{\s*([^}]*)\s*}/); - if (!match || !match[1].trim()) - return []; - let signature = match[1]; - return signature.split(',').map((t: string) => t.trim()); -} - function innerRegisterFixture(name: any, scope: string, fn: any, caller: Function) { const obj = {stack: ''}; Error.captureStackTrace(obj, caller); @@ -210,7 +228,7 @@ function innerRegisterFixture(name: any, scope: string, fn: any, caller: Functio registrationsByFile.get(file).push(registration); }; -export function registerFixture(name: T, fn: (params: FixtureParameters & WorkerState & TestState, test: (arg: TestState[T]) => Promise) => Promise) { +export function registerFixture(name: T, fn: (params: FixtureParameters & WorkerState & TestState, test: (arg: TestState[T]) => Promise) => Promise) { innerRegisterFixture(name, 'test', fn, registerFixture); }; diff --git a/test-runner/src/fixturesUI.js b/test-runner/src/fixturesUI.js index a37078c4d8383..a38887270b0eb 100644 --- a/test-runner/src/fixturesUI.js +++ b/test-runner/src/fixturesUI.js @@ -63,7 +63,7 @@ function fixturesUI(wrappers, suite) { if (suite.isPending()) fn = null; - const wrapper = fn ? wrappers.testWrapper(fn) : undefined; + const wrapper = fn ? wrappers.testWrapper(fn, title, file, specs.slow && specs.slow[0]) : undefined; if (wrapper) { wrapper.toString = () => fn.toString(); wrapper.__original = fn; @@ -72,8 +72,6 @@ function fixturesUI(wrappers, suite) { test.file = file; suite.addTest(test); const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0]; - if (specs.slow && specs.slow[0]) - test.timeout(90000); if (only) test.__only = true; if (!only && specs.skip && specs.skip[0]) diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index de4aea4e8c2db..da7ef20ecc48d 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -31,49 +31,55 @@ declare global { global.expect = require('expect'); const GoldenUtils = require('./GoldenUtils'); +export type TestRunnerEntry = { + file: string; + ordinals: number[]; + configuredFile: string; + configurationObject: any; +}; + class NullReporter {} export class TestRunner extends EventEmitter { mocha: any; - _currentOrdinal: number; - _failedWithError: boolean; - _file: any; - _ordinals: Set; - _remaining: Set; - _trialRun: any; - _passes: number; - _failures: number; - _pending: number; - _configuredFile: any; - _configurationObject: any; - _configurationString: any; - _parsedGeneratorConfiguration: {}; - _relativeTestFile: string; - _runner: any; - constructor(entry, options, workerId) { + private _currentOrdinal = -1; + private _failedWithError = false; + private _file: any; + private _ordinals: Set; + private _remaining: Set; + private _trialRun: any; + private _passes = 0; + private _failures = 0; + private _pending = 0; + private _configuredFile: any; + private _configurationObject: any; + private _parsedGeneratorConfiguration: any = {}; + private _relativeTestFile: string; + private _runner: Mocha.Runner; + private _outDir: string; + private _timeout: number; + private _testDir: string; + + constructor(entry: TestRunnerEntry, options, workerId) { super(); this.mocha = new Mocha({ reporter: NullReporter, - timeout: options.timeout, + timeout: 0, ui: fixturesUI.bind(null, { - testWrapper: fn => this._testWrapper(fn), + testWrapper: (fn, title, file, isSlow) => this._testWrapper(fn, title, file, isSlow), hookWrapper: (hook, fn) => this._hookWrapper(hook, fn), ignoreOnly: true }), }); - this._currentOrdinal = -1; - this._failedWithError = false; this._file = entry.file; this._ordinals = new Set(entry.ordinals); this._remaining = new Set(entry.ordinals); this._trialRun = options.trialRun; - this._passes = 0; - this._failures = 0; - this._pending = 0; + this._timeout = options.timeout; + this._testDir = options.testDir; + this._outDir = options.outputDir; this._configuredFile = entry.configuredFile; this._configurationObject = entry.configurationObject; - this._configurationString = entry.configurationString; - this._parsedGeneratorConfiguration = {}; for (const {name, value} of this._configurationObject) { this._parsedGeneratorConfiguration[name] = value; // @ts-ignore @@ -168,8 +174,15 @@ export class TestRunner extends EventEmitter { return true; } - _testWrapper(fn) { - const wrapped = fixturePool.wrapTestCallback(fn); + _testWrapper(fn, title, file, isSlow) { + const timeout = isSlow ? this._timeout * 3 : this._timeout; + const wrapped = fixturePool.wrapTestCallback(fn, timeout, { + outputDir: this._outDir, + testDir: this._testDir, + title, + file, + timeout + }); return wrapped ? (done, ...args) => { if (!this._shouldRunTest()) { done(); @@ -183,7 +196,7 @@ export class TestRunner extends EventEmitter { if (!this._shouldRunTest(true)) return; return hook(async () => { - return await fixturePool.resolveParametersAndRun(fn); + return await fixturePool.resolveParametersAndRun(fn, 0); }); } diff --git a/test/playwright.fixtures.ts b/test/playwright.fixtures.ts index 7678425e50bb7..2a79a22a6316d 100644 --- a/test/playwright.fixtures.ts +++ b/test/playwright.fixtures.ts @@ -41,9 +41,9 @@ declare global { browserType: BrowserType; browser: Browser; httpService: {server: TestServer, httpsServer: TestServer} + toImpl: (rpcObject: any) => any; } interface TestState { - toImpl: (rpcObject: any) => any; context: BrowserContext; server: TestServer; page: Page; @@ -142,7 +142,7 @@ registerWorkerFixture('playwright', async({browserName, wire}, test) => { }); -registerFixture('toImpl', async ({playwright}, test) => { +registerWorkerFixture('toImpl', async ({playwright}, test) => { await test((playwright as any)._toImpl); }); @@ -165,6 +165,14 @@ registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, te await browser.close(); }); +registerWorkerFixture('asset', async ({}, test) => { + await test(p => path.join(__dirname, `assets`, p)); +}); + +registerWorkerFixture('golden', async ({browserName}, test) => { + await test(p => path.join(browserName, p)); +}); + registerFixture('context', async ({browser}, test) => { const context = await browser.newContext(); await test(context); @@ -173,7 +181,13 @@ registerFixture('context', async ({browser}, test) => { registerFixture('page', async ({context}, test) => { const page = await context.newPage(); - await test(page); + const { success, info } = await test(page); + if (!success) { + const relativePath = path.relative(info.testDir, info.file).replace(/\.spec\.[jt]s/, ''); + const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_'); + const assetPath = path.join(info.outputDir, relativePath, sanitizedTitle) + '-failed.png'; + await page.screenshot({ path: assetPath }); + } }); registerFixture('server', async ({httpService}, test) => { @@ -192,14 +206,6 @@ registerFixture('tmpDir', async ({}, test) => { await removeFolderAsync(tmpDir).catch(e => {}); }); -registerWorkerFixture('asset', async ({}, test) => { - await test(p => path.join(__dirname, `assets`, p)); -}); - -registerWorkerFixture('golden', async ({browserName}, test) => { - await test(p => path.join(browserName, p)); -}); - export const options = { CHROMIUM: parameters.browserName === 'chromium', FIREFOX: parameters.browserName === 'firefox', diff --git a/test/test-runner.spec.ts b/test/test-runner.spec.ts new file mode 100644 index 0000000000000..d11bb7116a9e0 --- /dev/null +++ b/test/test-runner.spec.ts @@ -0,0 +1,49 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import './test-runner-helper'; +import { registerFixture } from '../test-runner'; + +declare global { + interface TestState { + a: string; + b: string; + ab: string; + zero: number; + tear: string; + } +} + +let zero = 0; +registerFixture('zero', async ({}, test) => { + await test(zero++); +}); + +registerFixture('a', async ({zero}, test) => { + await test('a' + zero); +}); + +registerFixture('b', async ({zero}, test) => { + await test('b' + zero); +}); + +registerFixture('ab', async ({a, b}, test) => { + await test(a + b); +}); + +it('should eval fixture once', async ({ab}) => { + expect(ab).toBe('a0b0'); +});