Skip to content

Commit

Permalink
test: mark some tests as skipped
Browse files Browse the repository at this point in the history
  • Loading branch information
@pavelfeldman
pavelfeldman committed Mar 2, 2020
1 parent 1186998 commit ba06fb2
Show file tree
Hide file tree
Showing 8 changed files with 104 additions and 80 deletions.
6 changes: 3 additions & 3 deletions test/accessibility.spec.js
View file
Expand Up @@ -140,7 +140,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT,
expect(await page.accessibility.snapshot()).toEqual(golden);
});
// WebKit rich text accessibility is iffy
!WEBKIT && it('rich text editable fields should have children', async function({page}) {
it.skip(WEBKIT)('rich text editable fields should have children', async function({page}) {
await page.setContent(`
<div contenteditable="true">
Edit this image: <img src="fakeimage.png" alt="my fake image">
Expand Down Expand Up @@ -171,7 +171,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT,
expect(snapshot.children[0]).toEqual(golden);
});
// WebKit rich text accessibility is iffy
!WEBKIT && it('rich text editable fields with role should have children', async function({page}) {
it.skip(WEBKIT)('rich text editable fields with role should have children', async function({page}) {
await page.setContent(`
<div contenteditable="true" role='textbox'>
Edit this image: <img src="fakeimage.png" alt="my fake image">
Expand Down Expand Up @@ -201,7 +201,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT,
});
// Firefox does not support contenteditable="plaintext-only".
// WebKit rich text accessibility is iffy
!FFOX && !WEBKIT && describe('plaintext contenteditable', function() {
describe.skip(FFOX || WEBKIT)('plaintext contenteditable', function() {
it('plain text field with role should not have children', async function({page}) {
await page.setContent(`
<div contenteditable="plaintext-only" role='textbox'>Edit this image:<img src="fakeimage.png" alt="my fake image"></div>`);
Expand Down
2 changes: 1 addition & 1 deletion test/navigation.spec.js
View file
Expand Up @@ -121,7 +121,7 @@ module.exports.describe = function({testRunner, expect, playwright, MAC, WIN, FF
});
await page.goto(server.PREFIX + '/frames/one-frame.html');
});
!WEBKIT && it('should fail when server returns 204', async({page, server}) => {
it.fail(WEBKIT)('should fail when server returns 204', async({page, server}) => {
// Webkit just loads an empty page.
server.setRoute('/empty.html', (req, res) => {
res.statusCode = 204;
Expand Down
2 changes: 1 addition & 1 deletion test/page.spec.js
View file
Expand Up @@ -652,7 +652,7 @@ module.exports.describe = function({testRunner, expect, headless, playwright, FF
expect(await page.evaluate(() => __injected)).toBe(42);
});

(CHROMIUM || FFOX) && it('should include sourceURL when path is provided', async({page, server}) => {
it.skip(WEBKIT)('should include sourceURL when path is provided', async({page, server}) => {
await page.goto(server.EMPTY_PAGE);
await page.addScriptTag({ path: path.join(__dirname, 'assets/injectedfile.js') });
const result = await page.evaluate(() => __injectedError.stack);
Expand Down
96 changes: 50 additions & 46 deletions test/playwright.spec.js
View file
Expand Up @@ -99,7 +99,7 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
ASSETS_DIR,
};

describe('Browser', function() {
describe('', function() {
beforeAll(async state => {
state.browser = await playwright.launch(defaultBrowserOptions);
state.browserServer = state.browser.__server__;
Expand Down Expand Up @@ -142,7 +142,7 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
await state.tearDown();
});

describe('Page', function() {
describe('', function() {
beforeEach(async state => {
state.context = await state.browser.newContext();
state.page = await state.context.newPage();
Expand All @@ -156,66 +156,70 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {

// Page-level tests that are given a browser, a context and a page.
// Each test is launched in a new browser context.
testRunner.loadTests(require('./accessibility.spec.js'), testOptions);
testRunner.loadTests(require('./click.spec.js'), testOptions);
testRunner.loadTests(require('./cookies.spec.js'), testOptions);
testRunner.loadTests(require('./dialog.spec.js'), testOptions);
testRunner.loadTests(require('./elementhandle.spec.js'), testOptions);
testRunner.loadTests(require('./emulation.spec.js'), testOptions);
testRunner.loadTests(require('./evaluation.spec.js'), testOptions);
testRunner.loadTests(require('./frame.spec.js'), testOptions);
testRunner.loadTests(require('./focus.spec.js'), testOptions);
testRunner.loadTests(require('./input.spec.js'), testOptions);
testRunner.loadTests(require('./jshandle.spec.js'), testOptions);
testRunner.loadTests(require('./keyboard.spec.js'), testOptions);
testRunner.loadTests(require('./mouse.spec.js'), testOptions);
testRunner.loadTests(require('./navigation.spec.js'), testOptions);
testRunner.loadTests(require('./network.spec.js'), testOptions);
testRunner.loadTests(require('./page.spec.js'), testOptions);
testRunner.loadTests(require('./queryselector.spec.js'), testOptions);
testRunner.loadTests(require('./screenshot.spec.js'), testOptions);
testRunner.loadTests(require('./waittask.spec.js'), testOptions);
testRunner.loadTests(require('./interception.spec.js'), testOptions);
testRunner.loadTests(require('./geolocation.spec.js'), testOptions);
testRunner.loadTests(require('./workers.spec.js'), testOptions);
testRunner.loadTests(require('./capabilities.spec.js'), testOptions);

if (CHROMIUM) {
describe('[Accessibility]', () => testRunner.loadTests(require('./accessibility.spec.js'), testOptions));
describe('[Driver]', () => {
testRunner.loadTests(require('./click.spec.js'), testOptions);
testRunner.loadTests(require('./cookies.spec.js'), testOptions);
testRunner.loadTests(require('./dialog.spec.js'), testOptions);
testRunner.loadTests(require('./elementhandle.spec.js'), testOptions);
testRunner.loadTests(require('./emulation.spec.js'), testOptions);
testRunner.loadTests(require('./evaluation.spec.js'), testOptions);
testRunner.loadTests(require('./frame.spec.js'), testOptions);
testRunner.loadTests(require('./focus.spec.js'), testOptions);
testRunner.loadTests(require('./input.spec.js'), testOptions);
testRunner.loadTests(require('./jshandle.spec.js'), testOptions);
testRunner.loadTests(require('./keyboard.spec.js'), testOptions);
testRunner.loadTests(require('./mouse.spec.js'), testOptions);
testRunner.loadTests(require('./navigation.spec.js'), testOptions);
testRunner.loadTests(require('./network.spec.js'), testOptions);
testRunner.loadTests(require('./page.spec.js'), testOptions);
testRunner.loadTests(require('./queryselector.spec.js'), testOptions);
testRunner.loadTests(require('./screenshot.spec.js'), testOptions);
testRunner.loadTests(require('./waittask.spec.js'), testOptions);
testRunner.loadTests(require('./interception.spec.js'), testOptions);
testRunner.loadTests(require('./geolocation.spec.js'), testOptions);
testRunner.loadTests(require('./workers.spec.js'), testOptions);
testRunner.loadTests(require('./capabilities.spec.js'), testOptions);
});

describe.skip(!CHROMIUM)('[Chromium]', () => {
testRunner.loadTests(require('./chromium/chromium.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/coverage.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/pdf.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/session.spec.js'), testOptions);
}
});

if (CHROMIUM || FFOX) {
describe('[Permissions]', () => {
testRunner.loadTests(require('./features/permissions.spec.js'), testOptions);
}

if (WEBKIT) {
testRunner.loadTests(require('./webkit/provisional.spec.js'), testOptions);
}
});
});

// Browser-level tests that are given a browser.
testRunner.loadTests(require('./browser.spec.js'), testOptions);
testRunner.loadTests(require('./browsercontext.spec.js'), testOptions);
testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions);
testRunner.loadTests(require('./popup.spec.js'), testOptions);
describe('[Driver]', () => {
testRunner.loadTests(require('./browser.spec.js'), testOptions);
testRunner.loadTests(require('./browsercontext.spec.js'), testOptions);
testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions);
testRunner.loadTests(require('./popup.spec.js'), testOptions);
});
});

// Top-level tests that launch Browser themselves.
testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions);
testRunner.loadTests(require('./fixtures.spec.js'), testOptions);
testRunner.loadTests(require('./launcher.spec.js'), testOptions);
testRunner.loadTests(require('./headful.spec.js'), testOptions);
testRunner.loadTests(require('./multiclient.spec.js'), testOptions);
describe('[Driver]', () => {
testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions);
testRunner.loadTests(require('./fixtures.spec.js'), testOptions);
testRunner.loadTests(require('./launcher.spec.js'), testOptions);
testRunner.loadTests(require('./headful.spec.js'), testOptions);
testRunner.loadTests(require('./multiclient.spec.js'), testOptions);
});

if (CHROMIUM) {
describe.skip(!CHROMIUM)('[Chromium]', () => {
testRunner.loadTests(require('./chromium/launcher.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/headful.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/oopif.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/tracing.spec.js'), testOptions);
}
});

testRunner.loadTests(require('./web.spec.js'), testOptions);
describe('[Driver]', () => {
testRunner.loadTests(require('./web.spec.js'), testOptions);
});
};
2 changes: 1 addition & 1 deletion test/test.js
View file
Expand Up @@ -125,7 +125,7 @@ new Reporter(testRunner, {
verbose: process.argv.includes('--verbose'),
summary: !process.argv.includes('--verbose'),
showSlowTests: process.env.CI ? 5 : 0,
showSkippedTests: 10,
showMarkedAsFailingTests: 10,
});

// await utils.initializeFlakinessDashboardIfNeeded(testRunner);
Expand Down
2 changes: 1 addition & 1 deletion test/web.spec.js
View file
Expand Up @@ -22,7 +22,7 @@ module.exports.describe = function({testRunner, expect, defaultBrowserOptions, p
const {it, fit, xit, dit} = testRunner;
const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;

(CHROMIUM || FFOX) && describe('Web SDK', function() {
describe('Web SDK', function() {
beforeAll(async state => {
state.controlledBrowserApp = await playwright.launchServer(defaultBrowserOptions);
state.hostBrowser = await playwright.launch(defaultBrowserOptions);
Expand Down
32 changes: 19 additions & 13 deletions utils/testrunner/Reporter.js
View file
Expand Up @@ -22,14 +22,14 @@ class Reporter {
constructor(runner, options = {}) {
const {
showSlowTests = 3,
showSkippedTests = Infinity,
showMarkedAsFailingTests = Infinity,
verbose = false,
summary = true,
} = options;
this._filePathToLines = new Map();
this._runner = runner;
this._showSlowTests = showSlowTests;
this._showSkippedTests = showSkippedTests;
this._showMarkedAsFailingTests = showMarkedAsFailingTests;
this._verbose = verbose;
this._summary = summary;
this._testCounter = 0;
Expand Down Expand Up @@ -110,16 +110,17 @@ class Reporter {
}

const skippedTests = this._runner.skippedTests();
if (this._showSkippedTests && this._summary && skippedTests.length) {
if (skippedTests.length > 0) {
console.log('\nSkipped:');
skippedTests.slice(0, this._showSkippedTests).forEach((test, index) => {
const markedAsFailingTests = this._runner.markedAsFailingTests();
if (this._showMarkedAsFailingTests && this._summary && markedAsFailingTests.length) {
if (markedAsFailingTests.length > 0) {
console.log('\nMarked as failing:');
markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => {
console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`);
});
}
if (this._showSkippedTests < skippedTests.length) {
if (this._showMarkedAsFailingTests < markedAsFailingTests.length) {
console.log('');
console.log(`... and ${colors.yellow(skippedTests.length - this._showSkippedTests)} more skipped tests ...`);
console.log(`... and ${colors.yellow(markedAsFailingTests.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`);
}
}

Expand All @@ -139,12 +140,14 @@ class Reporter {

const tests = this._runner.tests();
const executedTests = tests.filter(test => test.result);
const okTestsLength = executedTests.length - failedTests.length - skippedTests.length;
const okTestsLength = executedTests.length - failedTests.length - markedAsFailingTests.length - skippedTests.length;
let summaryText = '';
if (failedTests.length || skippedTests.length) {
if (failedTests.length || markedAsFailingTests.length) {
const summary = [`ok - ${colors.green(okTestsLength)}`];
if (failedTests.length)
summary.push(`failed - ${colors.red(failedTests.length)}`);
if (markedAsFailingTests.length)
summary.push(`marked as failing - ${colors.yellow(markedAsFailingTests.length)}`);
if (skippedTests.length)
summary.push(`skipped - ${colors.yellow(skippedTests.length)}`);
summaryText = ` (${summary.join(', ')})`;
Expand All @@ -167,9 +170,11 @@ class Reporter {
this._printVerboseTestResult(this._testCounter, test, workerId);
} else {
if (test.result === 'ok')
process.stdout.write(colors.green('.'));
process.stdout.write(colors.green('\u00B7'));
else if (test.result === 'skipped')
process.stdout.write(colors.yellow('*'));
process.stdout.write(colors.yellow('\u00B7'));
else if (test.result === 'markedAsFailing')
process.stdout.write(colors.yellow('\u00D7'));
else if (test.result === 'failed')
process.stdout.write(colors.red('F'));
else if (test.result === 'crashed')
Expand All @@ -192,7 +197,8 @@ class Reporter {
} else if (test.result === 'crashed') {
console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'skipped') {
console.log(`${prefix} ${colors.yellow('[SKIP]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'markedAsFailing') {
console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'timedout') {
console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'failed') {
Expand Down
42 changes: 28 additions & 14 deletions utils/testrunner/TestRunner.js
View file
Expand Up @@ -72,13 +72,14 @@ const TestMode = {
Run: 'run',
Skip: 'skip',
Focus: 'focus',
ExpectToFail: 'fail',
MarkAsFailing: 'markAsFailing',
Flake: 'flake'
};

const TestResult = {
Ok: 'ok',
ExpectToFail: 'skipped', // User marked as failed
MarkedAsFailing: 'markedAsFailing', // User marked as failed
Skipped: 'skipped', // User marked as skipped
Failed: 'failed', // Exception happened during running
TimedOut: 'timedout', // Timeout Exceeded while running
Terminated: 'terminated', // Execution terminated
Expand Down Expand Up @@ -138,7 +139,7 @@ class TestPass {
for (let suite = test.suite; suite; suite = suite.parentSuite)
this._workerDistribution.set(suite, workerId);
// Do not shard skipped tests across workers.
if (test.declaredMode !== TestMode.ExpectToFail)
if (test.declaredMode !== TestMode.MarkAsFailing && test.declaredMode !== TestMode.Skip)
workerId = (workerId + 1) % parallel;
}

Expand Down Expand Up @@ -200,8 +201,13 @@ class TestPass {
if (this._termination)
return;
this._runner._willStartTest(test, workerId);
if (test.declaredMode === TestMode.ExpectToFail) {
test.result = TestResult.ExpectToFail;
if (test.declaredMode === TestMode.MarkAsFailing) {
test.result = TestResult.MarkedAsFailing;
this._runner._didFinishTest(test, workerId);
return;
}
if (test.declaredMode === TestMode.Skip) {
test.result = TestResult.Skipped;
this._runner._didFinishTest(test, workerId);
return;
}
Expand Down Expand Up @@ -296,7 +302,7 @@ function specBuilder(action) {
};
func.fail = condition => {
if (condition)
mode = TestMode.ExpectToFail;
mode = TestMode.MarkAsFailing;
return func;
};
func.flake = condition => {
Expand Down Expand Up @@ -368,14 +374,20 @@ class TestRunner extends EventEmitter {
}

_addTest(name, callback, mode, timeout) {
if (mode === TestMode.Skip)
return;
let suite = this._currentSuite;
let expectToFail = suite.declaredMode === TestMode.ExpectToFail;
let markedAsFailing = suite.declaredMode === TestMode.MarkAsFailing;
while ((suite = suite.parentSuite))
markedAsFailing |= suite.declaredMode === TestMode.MarkAsFailing;
if (markedAsFailing)
mode = TestMode.MarkAsFailing;

suite = this._currentSuite;
let skip = suite.declaredMode === TestMode.Skip;
while ((suite = suite.parentSuite))
expectToFail |= suite.declaredMode === TestMode.ExpectToFail;
if (expectToFail)
mode = TestMode.ExpectToFail;
skip |= suite.declaredMode === TestMode.Skip;
if (skip)
mode = TestMode.Skip;

const test = new Test(this._currentSuite, name, callback, mode, timeout);
this._currentSuite.children.push(test);
this._tests.push(test);
Expand All @@ -384,8 +396,6 @@ class TestRunner extends EventEmitter {
}

_addSuite(mode, name, callback, ...args) {
if (mode === TestMode.Skip)
return;
const oldSuite = this._currentSuite;
const suite = new Suite(this._currentSuite, name, mode);
this._currentSuite.children.push(suite);
Expand Down Expand Up @@ -491,6 +501,10 @@ class TestRunner extends EventEmitter {
return this._tests.filter(test => test.result === 'skipped');
}

markedAsFailingTests() {
return this._tests.filter(test => test.result === 'markedAsFailing');
}

parallel() {
return this._parallel;
}
Expand Down

0 comments on commit ba06fb2

Please sign in to comment.