diff --git a/package.json b/package.json index f701c60d0b6b4..1dfdca32051ec 100644 --- a/package.json +++ b/package.json @@ -15,13 +15,14 @@ "ctestd": "cross-env BROWSER=chromium jest --reporters=./jest/dot.js --colors", "ftestd": "cross-env BROWSER=firefox jest --reporters=./jest/dot.js --colors", "wtestd": "cross-env BROWSER=webkit jest --reporters=./jest/dot.js --colors", + "nojest": "cross-env BROWSER=chromium node --unhandled-rejections=strict ./test/nojest/nojest.js", "test": "npm run ctest && npm run ftest && npm run wtest", "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src", "tsc": "tsc -p .", "tsc-installer": "tsc -p ./src/install/tsconfig.json", "doc": "node utils/doclint/cli.js", "doc-channel": "node utils/doclint/cli.js --channel", - "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js && node utils/testrunner/test/test.js", + "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js", "lint": "npm run eslint && npm run tsc && npm run doc && npm run doc-channel && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra", "debug-test": "node --inspect-brk test/test.js", "clean": "rimraf lib && rimraf types", diff --git a/test/jest/fixturePool.js b/test/harness/fixturePool.js similarity index 82% rename from test/jest/fixturePool.js rename to test/harness/fixturePool.js index 8810d421e5583..524b9fceb70e8 100644 --- a/test/jest/fixturePool.js +++ b/test/harness/fixturePool.js @@ -106,6 +106,25 @@ class FixturePool { params[n] = this.instances.get(n).value; return fn(params); } + + patchToEnableFixtures(object, name) { + const original = object[name]; + object[name] = fn => { + return original(async () => { + return await this.resolveParametersAndRun(fn); + }); + } + } + + wrapTestCallback(callback) { + return async() => { + try { + return await this.resolveParametersAndRun(callback); + } finally { + await this.teardownScope('test'); + } + }; + } } function fixtureParameterNames(fn) { @@ -117,8 +136,16 @@ function fixtureParameterNames(fn) { return signature.split(',').map(t => t.trim()); } -function registerFixture(name, scope, fn) { +function innerRegisterFixture(name, scope, fn) { registrations.set(name, { scope, fn }); -} +}; + +function registerFixture(name, fn) { + innerRegisterFixture(name, 'test', fn); +}; + +function registerWorkerFixture (name, fn) { + innerRegisterFixture(name, 'worker', fn); +}; -module.exports = { FixturePool, registerFixture }; +module.exports = { FixturePool, registerFixture, registerWorkerFixture }; diff --git a/test/jest/fixtures.js b/test/harness/fixtures.js similarity index 83% rename from test/jest/fixtures.js rename to test/harness/fixtures.js index 86155cf1cc2aa..4a821e16170da 100644 --- a/test/jest/fixtures.js +++ b/test/harness/fixtures.js @@ -18,20 +18,24 @@ const path = require('path'); const childProcess = require('child_process'); const playwrightImpl = require('../../index'); -const { TestServer } = require('../../utils/testserver/'); +const { TestServer } = require('../../utils/testserver'); const { Connection } = require('../../lib/rpc/client/connection'); const { Transport } = require('../../lib/rpc/transport'); const { setupInProcess } = require('../../lib/rpc/inprocess'); const { setUnderTest } = require('../../lib/helper'); +const { valueFromEnv } = require('./utils'); +const { registerFixture, registerWorkerFixture } = require('./fixturePool'); + setUnderTest(); const browserName = process.env.BROWSER || 'chromium'; module.exports = function registerFixtures(global) { - global.registerWorkerFixture('parallelIndex', async ({}, test) => { + registerWorkerFixture('parallelIndex', async ({}, test) => { await test(process.env.JEST_WORKER_ID - 1); }); - global.registerWorkerFixture('http_server', async ({parallelIndex}, test) => { + + registerWorkerFixture('http_server', async ({parallelIndex}, test) => { const assetsPath = path.join(__dirname, '..', 'assets'); const cachedPath = path.join(__dirname, '..', 'assets', 'cached'); @@ -59,7 +63,7 @@ module.exports = function registerFixtures(global) { ]); }); - global.registerWorkerFixture('defaultBrowserOptions', async({}, test) => { + registerWorkerFixture('defaultBrowserOptions', async({}, test) => { let executablePath = undefined; if (browserName === 'chromium' && process.env.CRPATH) executablePath = process.env.CRPATH; @@ -77,7 +81,7 @@ module.exports = function registerFixtures(global) { }); }); - global.registerWorkerFixture('playwright', async({}, test) => { + registerWorkerFixture('playwright', async({}, test) => { if (process.env.PWCHANNEL === 'wire') { const connection = new Connection(); const spawnedProcess = childProcess.fork(path.join(__dirname, '..', '..', 'lib', 'rpc', 'server'), [], { @@ -108,15 +112,15 @@ module.exports = function registerFixtures(global) { } }); - global.registerFixture('toImpl', async ({playwright}, test) => { + registerFixture('toImpl', async ({playwright}, test) => { await test(playwright._toImpl); }); - global.registerWorkerFixture('browserType', async ({playwright}, test) => { + registerWorkerFixture('browserType', async ({playwright}, test) => { await test(playwright[process.env.BROWSER || 'chromium']); }); - global.registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => { + registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => { const browser = await browserType.launch(defaultBrowserOptions); try { await test(browser); @@ -129,7 +133,7 @@ module.exports = function registerFixtures(global) { } }); - global.registerFixture('context', async ({browser}, test) => { + registerFixture('context', async ({browser}, test) => { const context = await browser.newContext(); try { await test(context); @@ -138,24 +142,18 @@ module.exports = function registerFixtures(global) { } }); - global.registerFixture('page', async ({context}, test) => { + registerFixture('page', async ({context}, test) => { const page = await context.newPage(); await test(page); }); - global.registerFixture('server', async ({http_server}, test) => { + registerFixture('server', async ({http_server}, test) => { http_server.server.reset(); await test(http_server.server); }); - global.registerFixture('httpsServer', async ({http_server}, test) => { + registerFixture('httpsServer', async ({http_server}, test) => { http_server.httpsServer.reset(); await test(http_server.httpsServer); }); } - -function valueFromEnv(name, defaultValue) { - if (!(name in process.env)) - return defaultValue; - return JSON.parse(process.env[name]); -} diff --git a/test/harness/testOptions.js b/test/harness/testOptions.js new file mode 100644 index 0000000000000..2f00f81031a2e --- /dev/null +++ b/test/harness/testOptions.js @@ -0,0 +1,38 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const os = require('os'); +const path = require('path'); +const { valueFromEnv } = require('./utils'); + +const platform = process.env.REPORT_ONLY_PLATFORM || os.platform(); +const browserName = process.env.BROWSER || 'chromium'; + +const testOptions = {}; +testOptions.MAC = platform === 'darwin'; +testOptions.LINUX = platform === 'linux'; +testOptions.WIN = platform === 'win32'; +testOptions.CHROMIUM = browserName === 'chromium'; +testOptions.FFOX = browserName === 'firefox'; +testOptions.WEBKIT = browserName === 'webkit'; +testOptions.USES_HOOKS = process.env.PWCHANNEL === 'wire'; +testOptions.CHANNEL = !!process.env.PWCHANNEL; +testOptions.HEADLESS = !!valueFromEnv('HEADLESS', true); +testOptions.ASSETS_DIR = path.join(__dirname, '..', 'assets'); +testOptions.GOLDEN_DIR = path.join(__dirname, '..', 'golden-' + browserName); +testOptions.OUTPUT_DIR = path.join(__dirname, '..', 'output-' + browserName); + +module.exports = testOptions; diff --git a/test/harness/utils.js b/test/harness/utils.js new file mode 100644 index 0000000000000..c080cd2e54002 --- /dev/null +++ b/test/harness/utils.js @@ -0,0 +1,23 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function valueFromEnv(name, defaultValue) { + if (!(name in process.env)) + return defaultValue; + return JSON.parse(process.env[name]); +} + +module.exports = { valueFromEnv }; diff --git a/test/jest/playwrightEnvironment.js b/test/jest/playwrightEnvironment.js index 023987b3ec2a8..9930d3fe6fc79 100644 --- a/test/jest/playwrightEnvironment.js +++ b/test/jest/playwrightEnvironment.js @@ -14,45 +14,29 @@ * limitations under the License. */ -const registerFixtures = require('./fixtures'); -const { FixturePool, registerFixture } = require('./fixturePool'); +const { FixturePool, registerFixture, registerWorkerFixture } = require('../harness/fixturePool'); +const registerFixtures = require('../harness/fixtures'); const os = require('os'); const path = require('path'); const fs = require('fs'); const debug = require('debug'); const util = require('util'); -const platform = process.env.REPORT_ONLY_PLATFORM || os.platform(); const GoldenUtils = require('../../utils/testrunner/GoldenUtils'); const {installCoverageHooks} = require('./coverage'); -const browserName = process.env.BROWSER || 'chromium'; const reportOnly = !!process.env.REPORT_ONLY_PLATFORM; const { ModuleMocker } = require('jest-mock'); -const testOptions = {}; -testOptions.MAC = platform === 'darwin'; -testOptions.LINUX = platform === 'linux'; -testOptions.WIN = platform === 'win32'; -testOptions.CHROMIUM = browserName === 'chromium'; -testOptions.FFOX = browserName === 'firefox'; -testOptions.WEBKIT = browserName === 'webkit'; -testOptions.USES_HOOKS = process.env.PWCHANNEL === 'wire'; -testOptions.CHANNEL = !!process.env.PWCHANNEL; -testOptions.HEADLESS = !!valueFromEnv('HEADLESS', true); -testOptions.ASSETS_DIR = path.join(__dirname, '..', 'assets'); -testOptions.GOLDEN_DIR = path.join(__dirname, '..', 'golden-' + browserName); -testOptions.OUTPUT_DIR = path.join(__dirname, '..', 'output-' + browserName); -global.testOptions = testOptions; - -global.registerFixture = (name, fn) => { - registerFixture(name, 'test', fn); -}; - -global.registerWorkerFixture = (name, fn) => { - registerFixture(name, 'worker', fn); -}; - +Error.stackTraceLimit = 15; +global.testOptions = require('../harness/testOptions'); +global.registerFixture = registerFixture; +global.registerWorkerFixture = registerWorkerFixture; registerFixtures(global); +const browserName = process.env.BROWSER || 'chromium'; + +const goldenPath = path.join(__dirname, '..', 'golden-' + browserName); +const outputPath = path.join(__dirname, '..', 'output-' + browserName); + let currentFixturePool = null; process.on('SIGINT', async () => { @@ -89,7 +73,7 @@ class PlaywrightEnvironment { this.uninstallCoverage(); const testRoot = path.join(__dirname, '..'); const relativeTestPath = path.relative(testRoot, this.testPath); - const coveragePath = path.join(this.global.testOptions.OUTPUT_DIR, 'coverage', relativeTestPath + '.json'); + const coveragePath = path.join(outputPath, 'coverage', relativeTestPath + '.json'); const coverageJSON = [...this.coverage.keys()].filter(key => this.coverage.get(key)); await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true }); await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8'); @@ -101,19 +85,10 @@ class PlaywrightEnvironment { return script.runInThisContext(); } - patchToEnableFixtures(object, name) { - const original = object[name]; - object[name] = fn => { - return original(async () => { - return await this.fixturePool.resolveParametersAndRun(fn); - }); - } - } - async handleTestEvent(event, state) { if (event.name === 'setup') { - this.patchToEnableFixtures(this.global, 'beforeEach'); - this.patchToEnableFixtures(this.global, 'afterEach'); + this.fixturePool.patchToEnableFixtures(this.global, 'beforeEach'); + this.fixturePool.patchToEnableFixtures(this.global, 'afterEach'); const describeSkip = this.global.describe.skip; this.global.describe.skip = (...args) => { @@ -152,7 +127,7 @@ class PlaywrightEnvironment { function toBeGolden(received, goldenName) { const {snapshotState} = this; const updateSnapshot = snapshotState._updateSnapshot; - const expectedPath = path.join(testOptions.GOLDEN_DIR, goldenName); + const expectedPath = path.join(goldenPath, goldenName); const fileExists = fs.existsSync(expectedPath); if (updateSnapshot === 'all' || (updateSnapshot === 'new' && !fileExists)) { fs.writeFileSync(expectedPath, received); @@ -166,8 +141,8 @@ class PlaywrightEnvironment { }; const {pass, message} = GoldenUtils.compare(received, { - goldenPath: testOptions.GOLDEN_DIR, - outputPath: testOptions.OUTPUT_DIR, + goldenPath, + outputPath, goldenName }); if (pass) @@ -210,12 +185,6 @@ class PlaywrightEnvironment { } } -function valueFromEnv(name, defaultValue) { - if (!(name in process.env)) - return defaultValue; - return JSON.parse(process.env[name]); -} - function testOrSuiteName(o) { if (o.name === 'ROOT_DESCRIBE_BLOCK') return ''; diff --git a/test/test.js b/test/nojest/nojest.js similarity index 50% rename from test/test.js rename to test/nojest/nojest.js index 6e7ea45a4065d..8ec58f20a873f 100644 --- a/test/test.js +++ b/test/nojest/nojest.js @@ -16,14 +16,24 @@ */ const fs = require('fs'); -const utils = require('./utils'); const path = require('path'); +const os = require('os'); const pirates = require('pirates'); const babel = require('@babel/core'); -const TestRunner = require('../utils/testrunner/'); -const { PlaywrightEnvironment, BrowserTypeEnvironment, BrowserEnvironment, PageEnvironment} = require('./environments.js'); +const TestRunner = require('../../utils/testrunner'); +const { FixturePool, registerFixture, registerWorkerFixture } = require('../harness/fixturePool'); +const registerFixtures = require('../harness/fixtures'); +const testOptions = require('../harness/testOptions'); Error.stackTraceLimit = 15; +global.testOptions = require('../harness/testOptions'); +global.registerFixture = registerFixture; +global.registerWorkerFixture = registerWorkerFixture; +registerFixtures(global); +process.env.JEST_WORKER_ID = 1; +const browserName = process.env.BROWSER || 'chromium'; +const goldenPath = path.join(__dirname, '..', 'golden-' + browserName); +const outputPath = path.join(__dirname, '..', 'output-' + browserName); function getCLIArgument(argName) { for (let i = 0; i < process.argv.length; ++i) { @@ -51,14 +61,11 @@ function collect(browserNames) { let timeout = process.env.CI ? 30 * 1000 : 10 * 1000; if (!isNaN(process.env.TIMEOUT)) timeout = parseInt(process.env.TIMEOUT * 1000, 10); - const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10); - if (MAJOR_NODEJS_VERSION >= 8 && require('inspector').url()) { + if (require('inspector').url()) { console.log('Detected inspector - disabling timeout to be debugger-friendly'); timeout = 0; } - const config = require('./test.config'); - const testRunner = new TestRunner({ timeout, totalTimeout: process.env.CI ? 30 * 60 * 1000 * browserNames.length : 0, // 30 minutes per browser on CI @@ -69,93 +76,52 @@ function collect(browserNames) { showSlowTests: process.env.CI ? 5 : 0, showMarkedAsFailingTests: 10, lineBreak: parseInt(getCLIArgument('--line-break') || 0, 10), + outputPath, + goldenPath }); - if (config.setupTestRunner) - config.setupTestRunner(testRunner); for (const [key, value] of Object.entries(testRunner.api())) global[key] = value; - // TODO: this should be a preinstalled playwright by default. - const playwrightPath = config.playwrightPath; - const playwright = require('..'); - const { setUnderTest } = require(require('path').join(playwrightPath, 'lib/helper.js')); - setUnderTest(); - - const playwrightEnvironment = new PlaywrightEnvironment(playwright); - testRunner.collector().useEnvironment(playwrightEnvironment); - for (const e of config.globalEnvironments || []) - testRunner.collector().useEnvironment(e); - - // TODO(rpc): do not use global playwright and browserType, rely solely on environments. - global.playwright = playwright; - - for (const browserName of browserNames) { - const browserType = playwright[browserName]; - const browserTypeEnvironment = new BrowserTypeEnvironment(browserName); - - // TODO: maybe launch options per browser? - const launchOptions = { - ...(config.launchOptions || {}), - handleSIGINT: false, - }; - if (launchOptions.executablePath) - launchOptions.executablePath = launchOptions.executablePath[browserName]; - if (launchOptions.executablePath) { - const YELLOW_COLOR = '\x1b[33m'; - const RESET_COLOR = '\x1b[0m'; - console.warn(`${YELLOW_COLOR}WARN: running ${browserName} tests with ${launchOptions.executablePath}${RESET_COLOR}`); - browserType._executablePath = launchOptions.executablePath; - delete launchOptions.executablePath; - } else { - if (!fs.existsSync(browserType.executablePath())) - throw new Error(`Browser is not downloaded. Run 'npm install' and try to re-run tests`); - } - - const browserEnvironment = new BrowserEnvironment(launchOptions, config.dumpLogOnFailure); - const pageEnvironment = new PageEnvironment(); - - const suiteName = { 'chromium': 'Chromium', 'firefox': 'Firefox', 'webkit': 'WebKit' }[browserName]; - describe(suiteName, () => { - // In addition to state, expose these two on global so that describes can access them. - global.browserType = browserType; - global.HEADLESS = !!launchOptions.headless; - - testRunner.collector().useEnvironment(browserTypeEnvironment); - - for (const spec of config.specs || []) { - const skip = spec.browsers && !spec.browsers.includes(browserName); - (skip ? xdescribe : describe)(spec.title || '', () => { - for (const e of spec.environments || ['page']) { - if (e === 'page') { - testRunner.collector().useEnvironment(browserEnvironment); - testRunner.collector().useEnvironment(pageEnvironment); - } else { - testRunner.collector().useEnvironment(e); - } - } - for (const file of spec.files || []) { - const revert = pirates.addHook((code, filename) => { - const result = babel.transformFileSync(filename, { - presets: [ - ['@babel/preset-env', {targets: {node: 'current'}}], - '@babel/preset-typescript'] - }); - return result.code; - }, { - exts: ['.ts'] - }); - require(file); - revert(); - delete require.cache[require.resolve(file)]; - } + const collector = testRunner.collector(); + collector.addTestModifier('skip', (t, condition) => condition && t.setSkipped(true)); + collector.addSuiteModifier('skip', (s, condition) => condition && s.setSkipped(true)); + collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); + collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); + collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3)); + collector.addTestAttribute('debug', t => TraceTestEnvironment.enableForTest(t)); + testRunner.api().fdescribe = testRunner.api().describe.only; + testRunner.api().xdescribe = testRunner.api().describe.skip(true); + testRunner.api().fit = testRunner.api().it.only; + testRunner.api().xit = testRunner.api().it.skip(true); + testRunner.api().dit = testRunner.api().it.only.debug; + + const fixturePool = new FixturePool(); + fixturePool.patchToEnableFixtures(global, 'beforeEach'); + fixturePool.patchToEnableFixtures(global, 'afterEach'); + collector.addTestCallbackWrapper(callback => fixturePool.wrapTestCallback(callback)); + + describe('', () => { + for (const name of fs.readdirSync('test')) { + const file = path.join(process.cwd(), 'test', name); + if (!name.includes('.spec.')) + continue; + const revert = pirates.addHook((code, filename) => { + const result = babel.transformFileSync(filename, { + presets: [ + ['@babel/preset-env', {targets: {node: 'current'}}], + '@babel/preset-typescript'] }); - } + return result.code; + }, { + exts: ['.ts'] + }); + require(file); + revert(); + delete require.cache[require.resolve(file)]; + } + }); - delete global.HEADLESS; - delete global.browserType; - }); - } for (const [key, value] of Object.entries(testRunner.api())) { // expect is used when running tests, while the rest of api is not. if (key !== 'expect') diff --git a/test/test.config.js b/test/test.config.js deleted file mode 100644 index 55e9e57d306fc..0000000000000 --- a/test/test.config.js +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright 2019 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const path = require('path'); -const utils = require('./utils'); -const {DefaultBrowserOptionsEnvironment, ServerEnvironment, GoldenEnvironment, TraceTestEnvironment} = require('./environments.js'); - -const playwrightPath = path.join(__dirname, '..'); - -const dumpLogOnFailure = valueFromEnv('DEBUGP', false); -const defaultBrowserOptionsEnvironment = new DefaultBrowserOptionsEnvironment({ - handleSIGINT: false, - slowMo: valueFromEnv('SLOW_MO', 0), - headless: !!valueFromEnv('HEADLESS', true), -}, dumpLogOnFailure, playwrightPath); - -const serverEnvironment = new ServerEnvironment(); -const customEnvironment = new GoldenEnvironment(); - -function valueFromEnv(name, defaultValue) { - if (!(name in process.env)) - return defaultValue; - return JSON.parse(process.env[name]); -} - -function setupTestRunner(testRunner) { - const collector = testRunner.collector(); - collector.addTestModifier('skip', (t, condition) => condition && t.setSkipped(true)); - collector.addSuiteModifier('skip', (s, condition) => condition && s.setSkipped(true)); - collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); - collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); - collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3)); - collector.addTestAttribute('debug', t => TraceTestEnvironment.enableForTest(t)); - testRunner.api().fdescribe = testRunner.api().describe.only; - testRunner.api().xdescribe = testRunner.api().describe.skip(true); - testRunner.api().fit = testRunner.api().it.only; - testRunner.api().xit = testRunner.api().it.skip(true); - testRunner.api().dit = testRunner.api().it.only.debug; -} - -module.exports = { - playwrightPath, - dumpLogOnFailure: valueFromEnv('DEBUGP', false), - launchOptions: { - executablePath: { - chromium: process.env.CRPATH, - firefox: process.env.FFPATH, - webkit: process.env.WKPATH, - }, - slowMo: valueFromEnv('SLOW_MO', 0), - headless: !!valueFromEnv('HEADLESS', true), - }, - - globalEnvironments: [defaultBrowserOptionsEnvironment, serverEnvironment], - setupTestRunner, - - specs: [ - { - files: [ - ], - environments: [customEnvironment], - }, - { - files: [ - ], - environments: [], - }, - ], -}; diff --git a/test/utils.js b/test/utils.js index 423b293189ce5..7fa60c74613be 100644 --- a/test/utils.js +++ b/test/utils.js @@ -189,27 +189,6 @@ const utils = module.exports = { await utils.removeFolderAsync(dir).catch(e => {}); }, - testOptions(browserType) { - const GOLDEN_DIR = path.join(__dirname, 'golden-' + browserType.name()); - const OUTPUT_DIR = path.join(__dirname, 'output-' + browserType.name()); - const ASSETS_DIR = path.join(__dirname, 'assets'); - return { - FFOX: browserType.name() === 'firefox', - WEBKIT: browserType.name() === 'webkit', - CHROMIUM: browserType.name() === 'chromium', - MAC: platform === 'darwin', - LINUX: platform === 'linux', - WIN: platform === 'win32', - browserType, - GOLDEN_DIR, - OUTPUT_DIR, - ASSETS_DIR, - USES_HOOKS: process.env.PWCHANNEL === 'wire', - CHANNEL: !!process.env.PWCHANNEL, - HEADLESS: !!valueFromEnv('HEADLESS', true), - }; - }, - setPlatform(p) { // To support isplaywrightready. platform = p; @@ -263,9 +242,3 @@ const utils = module.exports = { } }, }; - -function valueFromEnv(name, defaultValue) { - if (!(name in process.env)) - return defaultValue; - return JSON.parse(process.env[name]); -} diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js index eaeba36ccc2ab..ee17e5f9f672a 100644 --- a/utils/doclint/check_public_api/test/test.js +++ b/utils/doclint/check_public_api/test/test.js @@ -22,7 +22,10 @@ const mdBuilder = require('../MDBuilder'); const jsBuilder = require('../JSBuilder'); const TestRunner = require('../../../testrunner/'); -const runner = new TestRunner(); +const runner = new TestRunner({ + goldenPath: __dirname, + outputPath: __dirname +}); const {describe, xdescribe, fdescribe} = runner.api(); const {it, fit, xit} = runner.api(); @@ -66,14 +69,14 @@ async function testLint(state, testRun) { const jsSources = await Source.readdir(dirPath, '.js'); const messages = await checkPublicAPI(page, mdSources, jsSources.concat(tsSources)); const errors = messages.map(message => message.text); - expect(errors.join('\n')).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'}); + expect(errors.join('\n')).toBeGolden(path.join(testRun.test().name(), 'result.txt')); } async function testMDBuilder(state, testRun) { const dirPath = path.join(__dirname, testRun.test().name()); const sources = await Source.readdir(dirPath, '.md'); const {documentation} = await mdBuilder(page, sources); - expect(serialize(documentation)).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'}); + expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt')); } async function testJSBuilder(state, testRun) { @@ -81,7 +84,7 @@ async function testJSBuilder(state, testRun) { const jsSources = await Source.readdir(dirPath, '.js'); const tsSources = await Source.readdir(dirPath, '.ts'); const {documentation} = await jsBuilder.checkSources(jsSources.concat(tsSources)); - expect(serialize(documentation)).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'}); + expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt')); } /** diff --git a/utils/testrunner/Matchers.js b/utils/testrunner/Matchers.js index 5eab01fe4a862..39e0a34711a52 100644 --- a/utils/testrunner/Matchers.js +++ b/utils/testrunner/Matchers.js @@ -20,15 +20,88 @@ const Diff = require('text-diff'); const GoldenUtils = require('./GoldenUtils'); class Matchers { - constructor(customMatchers = {}) { - this._matchers = {}; - Object.assign(this._matchers, DefaultMatchers); - Object.assign(this._matchers, customMatchers); + constructor(config) { this.expect = this.expect.bind(this); - } - addMatcher(name, matcher) { - this._matchers[name] = matcher; + this._matchers = { + toBe: function(received, expected, message) { + message = message || `${received} == ${expected}`; + return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) }; + }, + + toBeFalsy: function(received, message) { + message = message || `${received}`; + return { pass: !received, message }; + }, + + toBeTruthy: function(received, message) { + message = message || `${received}`; + return { pass: !!received, message }; + }, + + toBeGreaterThan: function(received, other, message) { + message = message || `${received} > ${other}`; + return { pass: received > other, message }; + }, + + toBeGreaterThanOrEqual: function(received, other, message) { + message = message || `${received} >= ${other}`; + return { pass: received >= other, message }; + }, + + toBeLessThan: function(received, other, message) { + message = message || `${received} < ${other}`; + return { pass: received < other, message }; + }, + + toBeLessThanOrEqual: function(received, other, message) { + message = message || `${received} <= ${other}`; + return { pass: received <= other, message }; + }, + + toBeNull: function(received, message) { + message = message || `${received} == null`; + return { pass: received === null, message }; + }, + + toContain: function(received, other, message) { + message = message || `${received} ⊇ ${other}`; + return { pass: received.includes(other), message }; + }, + + toEqual: function(received, other, message) { + let receivedJson = stringify(received); + let otherJson = stringify(other); + let formatter = objectFormatter.bind(null, receivedJson, otherJson); + if (receivedJson.length < 40 && otherJson.length < 40) { + receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' '); + otherJson = otherJson.split('\n').map(line => line.trim()).join(' '); + formatter = stringFormatter.bind(null, receivedJson, otherJson); + } + message = message || `\n${receivedJson} ≈ ${otherJson}`; + return { pass: receivedJson === otherJson, message, formatter }; + }, + + toBeCloseTo: function(received, other, precision, message) { + return { + pass: Math.abs(received - other) < Math.pow(10, -precision), + message + }; + }, + + toBeInstanceOf: function(received, other, message) { + message = message || `${received.constructor.name} instanceof ${other.name}`; + return { pass: received instanceof other, message }; + }, + + toBeGolden: function(received, goldenName) { + return GoldenUtils.compare(received, { + goldenPath: config.goldenPath, + outputPath: config.outputPath, + goldenName + }); + } + } } expect(received) { @@ -155,82 +228,6 @@ function toBeFormatter(received, expected) { ].join('\n'); } -const DefaultMatchers = { - toBe: function(received, expected, message) { - message = message || `${received} == ${expected}`; - return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) }; - }, - - toBeFalsy: function(received, message) { - message = message || `${received}`; - return { pass: !received, message }; - }, - - toBeTruthy: function(received, message) { - message = message || `${received}`; - return { pass: !!received, message }; - }, - - toBeGreaterThan: function(received, other, message) { - message = message || `${received} > ${other}`; - return { pass: received > other, message }; - }, - - toBeGreaterThanOrEqual: function(received, other, message) { - message = message || `${received} >= ${other}`; - return { pass: received >= other, message }; - }, - - toBeLessThan: function(received, other, message) { - message = message || `${received} < ${other}`; - return { pass: received < other, message }; - }, - - toBeLessThanOrEqual: function(received, other, message) { - message = message || `${received} <= ${other}`; - return { pass: received <= other, message }; - }, - - toBeNull: function(received, message) { - message = message || `${received} == null`; - return { pass: received === null, message }; - }, - - toContain: function(received, other, message) { - message = message || `${received} ⊇ ${other}`; - return { pass: received.includes(other), message }; - }, - - toEqual: function(received, other, message) { - let receivedJson = stringify(received); - let otherJson = stringify(other); - let formatter = objectFormatter.bind(null, receivedJson, otherJson); - if (receivedJson.length < 40 && otherJson.length < 40) { - receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' '); - otherJson = otherJson.split('\n').map(line => line.trim()).join(' '); - formatter = stringFormatter.bind(null, receivedJson, otherJson); - } - message = message || `\n${receivedJson} ≈ ${otherJson}`; - return { pass: receivedJson === otherJson, message, formatter }; - }, - - toBeCloseTo: function(received, other, precision, message) { - return { - pass: Math.abs(received - other) < Math.pow(10, -precision), - message - }; - }, - - toBeInstanceOf: function(received, other, message) { - message = message || `${received.constructor.name} instanceof ${other.name}`; - return { pass: received instanceof other, message }; - }, - - toBeGolden: function(received, golden) { - return GoldenUtils.compare(received, golden); - }, -}; - function stringify(value) { function stabilize(key, object) { if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object)) diff --git a/utils/testrunner/TestCollector.js b/utils/testrunner/TestCollector.js index 7f95eefcf5119..ec522994030fb 100644 --- a/utils/testrunner/TestCollector.js +++ b/utils/testrunner/TestCollector.js @@ -172,6 +172,7 @@ class TestCollector { this._suiteAttributes = new Map(); this._testModifiers = new Map(); this._testAttributes = new Map(); + this._testCallbackWrappers = []; this._api = {}; this._currentSuite = new Suite(null, '', new Location()); @@ -189,6 +190,8 @@ class TestCollector { }); this._api.it = specBuilder(this._testModifiers, this._testAttributes, (specs, name, testCallback) => { const location = Location.getCallerLocation(); + for (const wrapper of this._testCallbackWrappers) + testCallback = wrapper(testCallback); const test = new Test(this._currentSuite, name, testCallback, location); test.setTimeout(timeout); for (const { callback, args } of specs) @@ -207,6 +210,10 @@ class TestCollector { return this._currentSuite.addEnvironment(environment); } + addTestCallbackWrapper(wrapper) { + this._testCallbackWrappers.push(wrapper); + } + addTestModifier(name, callback) { this._testModifiers.set(name, callback); } diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js index ca27554feeb3a..1cf2e5f40bdcd 100644 --- a/utils/testrunner/index.js +++ b/utils/testrunner/index.js @@ -40,6 +40,8 @@ class DefaultTestRunner { verbose, summary, lineBreak, + goldenPath, + outputPath, } = options; this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; @@ -61,7 +63,7 @@ class DefaultTestRunner { this._api = { ...this._collector.api(), - expect: new Matchers().expect, + expect: new Matchers({ goldenPath, outputPath }).expect, }; this._collector.addSuiteAttribute('only', s => this._filter.focusSuite(s)); this._collector.addSuiteAttribute('skip', s => s.setSkipped(true));