Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add repeat method to tests #2652

Merged
merged 10 commits into from
Apr 15, 2023
12 changes: 12 additions & 0 deletions docs/api/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,20 @@ type Awaitable<T> = T | PromiseLike<T>
type TestFunction = () => Awaitable<void>

interface TestOptions {
/**
* Will fail the test if it takes too long to execute
*/
timeout?: number
/**
* Will retry the test specific number of times if it fails
*/
retry?: number
/**
* Will repeat the same test several times even if it fails each time
* If you have "retry" option and it fails, it will use every retry in each cycle
* Useful for debugging random failings
*/
repeats?: number
}
```

Expand Down
126 changes: 67 additions & 59 deletions packages/runner/src/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,55 +125,63 @@ export async function runTest(test: Test, runner: VitestRunner) {

setCurrentTest(test)

const retry = test.retry || 1
for (let retryCount = 0; retryCount < retry; retryCount++) {
let beforeEachCleanups: HookCleanupCallback[] = []
try {
await runner.onBeforeTryTest?.(test, retryCount)
const repeats = typeof test.repeats === 'number' ? test.repeats : 1

beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
const retry = test.retry || 1

test.result.retryCount = retryCount
for (let retryCount = 0; retryCount < retry; retryCount++) {
let beforeEachCleanups: HookCleanupCallback[] = []
try {
await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount })

if (runner.runTest) {
await runner.runTest(test)
}
else {
const fn = getFn(test)
if (!fn)
throw new Error('Test function is not found. Did you add it using `setFn`?')
await fn()
}
test.result.retryCount = retryCount
test.result.repeatCount = repeatCount

// some async expect will be added to this array, in case user forget to await theme
if (test.promises) {
const result = await Promise.allSettled(test.promises)
const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean)
if (errors.length)
throw errors
}
beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])

await runner.onAfterTryTest?.(test, retryCount)
if (runner.runTest) {
await runner.runTest(test)
}
else {
const fn = getFn(test)
if (!fn)
throw new Error('Test function is not found. Did you add it using `setFn`?')
await fn()
}

test.result.state = 'pass'
}
catch (e) {
failTask(test.result, e)
}
// some async expect will be added to this array, in case user forget to await theme
if (test.promises) {
const result = await Promise.allSettled(test.promises)
const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean)
if (errors.length)
throw errors
}

try {
await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
await callCleanupHooks(beforeEachCleanups)
}
catch (e) {
failTask(test.result, e)
}
await runner.onAfterTryTest?.(test, { retry: retryCount, repeats: repeatCount })

if (test.result.state === 'pass')
break
if (!test.repeats)
test.result.state = 'pass'
else if (test.repeats && retry === retryCount)
test.result.state = 'pass'
}
catch (e) {
failTask(test.result, e)
}

// update retry info
updateTask(test, runner)
try {
await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
await callCleanupHooks(beforeEachCleanups)
}
catch (e) {
failTask(test.result, e)
}

if (test.result.state === 'pass')
break
// update retry info
updateTask(test, runner)
}
}

if (test.result.state === 'fail')
Expand Down Expand Up @@ -291,30 +299,30 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
catch (e) {
failTask(suite.result, e)
}
}

suite.result.duration = now() - start

if (suite.mode === 'run') {
if (!hasTests(suite)) {
suite.result.state = 'fail'
if (!suite.result.error) {
const error = processError(new Error(`No test found in suite ${suite.name}`))
suite.result.error = error
suite.result.errors = [error]
if (suite.mode === 'run') {
if (!hasTests(suite)) {
suite.result.state = 'fail'
if (!suite.result.error) {
const error = processError(new Error(`No test found in suite ${suite.name}`))
suite.result.error = error
suite.result.errors = [error]
}
}
else if (hasFailed(suite)) {
suite.result.state = 'fail'
}
else {
suite.result.state = 'pass'
}
}
else if (hasFailed(suite)) {
suite.result.state = 'fail'
}
else {
suite.result.state = 'pass'
}
}

await runner.onAfterRunSuite?.(suite)
updateTask(suite, runner)

updateTask(suite, runner)
suite.result.duration = now() - start

await runner.onAfterRunSuite?.(suite)
}
}

async function runSuiteChild(c: Task, runner: VitestRunner) {
Expand Down
13 changes: 13 additions & 0 deletions packages/runner/src/suite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,14 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
if (typeof options === 'number')
options = { timeout: options }

// inherit repeats and retry from suite
if (typeof suiteOptions === 'object') {
options = {
...suiteOptions,
...options,
}
}

const test: Test = {
id: '',
type: 'test',
Expand All @@ -73,6 +81,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
suite: undefined!,
fails: this.fails,
retry: options?.retry,
repeats: options?.repeats,
} as Omit<Test, 'context'> as Test

if (this.concurrent || concurrent)
Expand Down Expand Up @@ -124,6 +133,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
}

function initSuite() {
if (typeof suiteOptions === 'number')
suiteOptions = { timeout: suiteOptions }

suite = {
id: '',
type: 'suite',
Expand All @@ -132,6 +144,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
shuffle,
tasks: [],
}

setHooks(suite, createSuiteHooks())
}

Expand Down
4 changes: 2 additions & 2 deletions packages/runner/src/types/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,15 @@ export interface VitestRunner {
/**
* Called before actually running the test function. Already has "result" with "state" and "startTime".
*/
onBeforeTryTest?(test: Test, retryCount: number): unknown
onBeforeTryTest?(test: Test, options: { retry: number; repeats: number }): unknown
/**
* Called after result and state are set.
*/
onAfterRunTest?(test: Test): unknown
/**
* Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws.
*/
onAfterTryTest?(test: Test, retryCount: number): unknown
onAfterTryTest?(test: Test, options: { retry: number; repeats: number }): unknown

/**
* Called before running a single suite. Doesn't have "result" yet.
Expand Down
8 changes: 8 additions & 0 deletions packages/runner/src/types/tasks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export interface TaskBase {
result?: TaskResult
retry?: number
meta?: any
repeats?: number
}

export interface TaskCustom extends TaskBase {
Expand All @@ -35,6 +36,7 @@ export interface TaskResult {
htmlError?: string
hooks?: Partial<Record<keyof SuiteHooks, TaskState>>
retryCount?: number
repeatCount?: number
}

export type TaskResultPack = [id: string, result: TaskResult | undefined]
Expand Down Expand Up @@ -165,6 +167,12 @@ export interface TestOptions {
* @default 1
*/
retry?: number
/**
* How many times the test will repeat.
*
* @default 5
*/
repeats?: number
}

export type TestAPI<ExtraContext = {}> = ChainableTestAPI<ExtraContext> & {
Expand Down
3 changes: 3 additions & 0 deletions packages/vitest/src/node/reporters/renderers/listRenderer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
if (task.mode === 'skip' || task.mode === 'todo')
suffix += ` ${c.dim(c.gray('[skipped]'))}`

if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1)
suffix += c.yellow(` (repeat x${task.result.repeatCount})`)

if (task.result?.duration != null) {
if (task.result.duration > DURATION_LONG)
suffix += c.yellow(` ${Math.round(task.result.duration)}${c.dim('ms')}`)
Expand Down
50 changes: 50 additions & 0 deletions test/core/test/repeats.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { afterAll, describe, expect, test } from 'vitest'
sheremet-va marked this conversation as resolved.
Show resolved Hide resolved

const testNumbers: number[] = []

describe('testing it/test', () => {
const result = [1, 1, 1, 1, 1, 2, 2, 2]

test('test 1', () => {
testNumbers.push(1)
}, { repeats: 5 })

test('test 2', () => {
testNumbers.push(2)
}, { repeats: 3 })

test.fails('test 3', () => {
testNumbers.push(3)
expect(testNumbers).toStrictEqual(result)
}, { repeats: 1 })

afterAll(() => {
result.push(3)
expect(testNumbers).toStrictEqual(result)
})
})

const describeNumbers: number[] = []

describe('testing describe', () => {
test('test 1', () => {
describeNumbers.push(1)
})
}, { repeats: 3 })

afterAll(() => {
expect(describeNumbers).toStrictEqual([1, 1, 1])
})

const retryNumbers: number[] = []

describe('testing repeats with retry', () => {
const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
test('test 1', () => {
retryNumbers.push(1)
}, { repeats: 5, retry: 2 })

afterAll(() => {
expect(retryNumbers).toStrictEqual(result)
})
})
3 changes: 3 additions & 0 deletions test/reporters/tests/__snapshots__/html.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ exports[`html reporter > resolves to "failing" status for test file "json-fail"
"afterEach": "pass",
"beforeEach": "pass",
},
"repeatCount": 0,
"retryCount": 0,
"startTime": 0,
"state": "fail",
Expand Down Expand Up @@ -155,6 +156,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing
"afterEach": "pass",
"beforeEach": "pass",
},
"repeatCount": 0,
"retryCount": 0,
"startTime": 0,
"state": "pass",
Expand Down Expand Up @@ -201,6 +203,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing
"afterEach": "pass",
"beforeEach": "pass",
},
"repeatCount": 0,
"retryCount": 0,
"startTime": 0,
"state": "pass",
Expand Down