Skip to content

Commit

Permalink
feat(coverage): add reportOnFailure option
Browse files Browse the repository at this point in the history
  • Loading branch information
AriPerkkio committed May 26, 2023
1 parent 56b9927 commit f8de8f8
Show file tree
Hide file tree
Showing 9 changed files with 82 additions and 32 deletions.
9 changes: 9 additions & 0 deletions docs/config/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -778,6 +778,15 @@ The reporter has three different types:

Since Vitest 0.31.0, you can check your coverage report in Vitest UI: check [Vitest UI Coverage](/guide/coverage#vitest-ui) for more details.

#### coverage.reportOnFailure

- **Type:** `boolean`
- **Default:** `true`
- **Available for providers:** `'c8' | 'istanbul'`
- **CLI:** `--coverage.reportOnFailure`, `--coverage.reportOnFailure=false`

Generate coverage report even when tests fail.

#### coverage.skipFull

- **Type:** `boolean`
Expand Down
1 change: 1 addition & 0 deletions packages/vitest/src/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export const coverageConfigDefaults: ResolvedCoverageOptions = {
cleanOnRerun: true,
reportsDirectory: './coverage',
exclude: defaultCoverageExcludes,
reportOnFailure: true,
reporter: [['text', {}], ['html', {}], ['clover', {}], ['json', {}]],
// default extensions used by c8, plus '.vue' and '.svelte'
// see https://github.com/istanbuljs/schema/blob/master/default-extension.js
Expand Down
3 changes: 3 additions & 0 deletions packages/vitest/src/node/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -692,6 +692,9 @@ export class Vitest {
}

private async reportCoverage(allTestsRun: boolean) {
if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0)
return

if (this.coverageProvider) {
this.logger.log(c.blue(' % ') + c.dim('Coverage report from ') + c.yellow(this.coverageProvider.name))
await this.coverageProvider.reportCoverage({ allTestsRun })
Expand Down
8 changes: 8 additions & 0 deletions packages/vitest/src/types/coverage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ type FieldsWithDefaultValues =
| 'reportsDirectory'
| 'exclude'
| 'extension'
| 'reportOnFailure'

export type ResolvedCoverageOptions<T extends Provider = Provider> =
& CoverageOptions<T>
Expand Down Expand Up @@ -208,6 +209,13 @@ export interface BaseCoverageOptions {
* @default false
*/
thresholdAutoUpdate?: boolean

/**
* Generate coverage report even when tests fail.
*
* @default true
*/
reportOnFailure?: boolean
}

export interface CoverageIstanbulOptions extends BaseCoverageOptions {
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions test/coverage-test/test/configuration-options.test-d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ test('provider module', () => {
extension: ['string'],
reporter: [['html', {}], ['json', { file: 'string' }]],
reportsDirectory: 'string',
reportOnFailure: true,
}
},
clean(_: boolean) {},
Expand Down
1 change: 1 addition & 0 deletions test/fails/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"coverage": "vitest run --coverage"
},
"devDependencies": {
"@vitest/coverage-istanbul": "workspace:*",
"jsdom": "^21.0.0",
"vitest": "workspace:*"
}
Expand Down
26 changes: 13 additions & 13 deletions test/fails/test/__snapshots__/runner.test.ts.snap
Original file line number Diff line number Diff line change
@@ -1,44 +1,44 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html

exports[`should fails > .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
exports[`should fail .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;

exports[`should fails > each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
exports[`should fail each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;

exports[`should fails > empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file <rootDir>/empty.test.ts"`;
exports[`should fail empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file <rootDir>/empty.test.ts"`;
exports[`should fails > expect.test.ts > expect.test.ts 1`] = `"AssertionError: expected 2 to deeply equal 3"`;
exports[`should fail expect.test.ts > expect.test.ts 1`] = `"AssertionError: expected 2 to deeply equal 3"`;
exports[`should fails > hook-timeout.test.ts > hook-timeout.test.ts 1`] = `"Error: Hook timed out in 10ms."`;
exports[`should fail hook-timeout.test.ts > hook-timeout.test.ts 1`] = `"Error: Hook timed out in 10ms."`;
exports[`should fails > hooks-called.test.ts > hooks-called.test.ts 1`] = `
exports[`should fail hooks-called.test.ts > hooks-called.test.ts 1`] = `
"Error: after all
Error: before all"
`;
exports[`should fails > inline-snapshop-inside-each.test.ts > inline-snapshop-inside-each.test.ts 1`] = `
exports[`should fail inline-snapshop-inside-each.test.ts > inline-snapshop-inside-each.test.ts 1`] = `
"Error: InlineSnapshot cannot be used inside of test.each or describe.each
Error: InlineSnapshot cannot be used inside of test.each or describe.each
Error: InlineSnapshot cannot be used inside of test.each or describe.each
Error: InlineSnapshot cannot be used inside of test.each or describe.each
Error: InlineSnapshot cannot be used inside of test.each or describe.each"
`;
exports[`should fails > mock-import-proxy-module.test.ts > mock-import-proxy-module.test.ts 1`] = `"Error: There are some problems in resolving the mocks API."`;
exports[`should fail mock-import-proxy-module.test.ts > mock-import-proxy-module.test.ts 1`] = `"Error: There are some problems in resolving the mocks API."`;
exports[`should fails > nested-suite.test.ts > nested-suite.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
exports[`should fail nested-suite.test.ts > nested-suite.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
exports[`should fails > primitive-error.test.ts > primitive-error.test.ts 1`] = `"Unknown Error: 42"`;
exports[`should fail primitive-error.test.ts > primitive-error.test.ts 1`] = `"Unknown Error: 42"`;
exports[`should fails > stall.test.ts > stall.test.ts 1`] = `
exports[`should fail stall.test.ts > stall.test.ts 1`] = `
"TypeError: failure
TypeError: failure
TypeError: failure
TypeError: failure"
`;
exports[`should fails > test-timeout.test.ts > test-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
exports[`should fail test-timeout.test.ts > test-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
exports[`should fails > unhandled.test.ts > unhandled.test.ts 1`] = `
exports[`should fail unhandled.test.ts > unhandled.test.ts 1`] = `
"Error: some error
Error: Uncaught [Error: some error]"
`;
62 changes: 43 additions & 19 deletions test/fails/test/runner.test.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,49 @@
import { resolve } from 'pathe'
import fg from 'fast-glob'
import { describe, expect, it } from 'vitest'
import { expect, it } from 'vitest'

import { runVitest } from '../../test-utils'

describe('should fails', async () => {
const root = resolve(__dirname, '../fixtures')
const files = await fg('**/*.test.ts', { cwd: root, dot: true })

for (const file of files) {
it(file, async () => {
const { stderr } = await runVitest({ root }, [file])

expect(stderr).toBeTruthy()
const msg = String(stderr)
.split(/\n/g)
.reverse()
.filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest'))
.map(i => i.trim().replace(root, '<rootDir>'),
).join('\n')
expect(msg).toMatchSnapshot(file)
}, 30_000)
}
const root = resolve(__dirname, '../fixtures')
const files = await fg('**/*.test.ts', { cwd: root, dot: true })

it.each(files)('should fail %s', async (file) => {
const { stderr } = await runVitest({ root }, [file])

expect(stderr).toBeTruthy()
const msg = String(stderr)
.split(/\n/g)
.reverse()
.filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest'))
.map(i => i.trim().replace(root, '<rootDir>'),
).join('\n')
expect(msg).toMatchSnapshot(file)
}, 30_000)

it('should report coverage when "coverag.reportOnFailure: true" and tests fail', async () => {
const { stdout } = await runVitest({
root,
coverage: {
enabled: true,
provider: 'istanbul',
reportOnFailure: true,
reporter: ['text'],
},
}, [files[0]])

expect(stdout).toMatch('Coverage report from istanbul')
})

it('should not report coverage when "coverag.reportOnFailure: false" and tests fail', async () => {
const { stdout } = await runVitest({
root,
coverage: {
enabled: true,
provider: 'istanbul',
reportOnFailure: false,
reporter: ['text'],
},
}, [files[0]])

expect(stdout).not.toMatch('Coverage report from istanbul')
})

0 comments on commit f8de8f8

Please sign in to comment.