feat: add repeat method to tests

docs/api/index.md

@@ -288,6 +288,21 @@ You cannot use this syntax, when using Vitest as [type checker](/guide/testing-t
 You cannot use this syntax, when using Vitest as [type checker](/guide/testing-types).
 :::
 
+### test.repeats
+
+- **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void`
+- **Alias:** `it.repeats`
+
+  If you want to run a test multiple times to see if it passes on all attempts, you can use `test.repeats` to do so. Without the `repeats` options it will only run once.
+
+  ```ts
+  import { expect, test } from 'vitest'
+
+  test.repeats('repeated test', () => {
+    expect(true).toBe(true)
+  }, { repeats: 3 })
+  ```
+
 ## bench
 
 - **Type:** `(name: string, fn: BenchFunction, options?: BenchOptions) => void`

packages/runner/src/run.ts

@@ -125,47 +125,55 @@ export async function runTest(test: Test, runner: VitestRunner) {
 
   setCurrentTest(test)
 
-  const retry = test.retry || 1
-  for (let retryCount = 0; retryCount < retry; retryCount++) {
-    let beforeEachCleanups: HookCleanupCallback[] = []
-    try {
-      await runner.onBeforeTryTest?.(test, retryCount)
+  const repeats = test.repeats ? test.repeats || 1 : 1
 
-      beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
+  for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
+    const retry = test.retry || 1
 
-      test.result.retryCount = retryCount
+    for (let retryCount = 0; retryCount < retry; retryCount++) {
+      let beforeEachCleanups: HookCleanupCallback[] = []
+      try {
+        await runner.onBeforeTryTest?.(test, retryCount)
 
-      if (runner.runTest) {
-        await runner.runTest(test)
-      }
-      else {
-        const fn = getFn(test)
-        if (!fn)
-          throw new Error('Test function is not found. Did you add it using `setFn`?')
-        await fn()
-      }
+        beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
 
-      await runner.onAfterTryTest?.(test, retryCount)
+        test.result.retryCount = retryCount
+        test.result.repeatCount = repeatCount
 
-      test.result.state = 'pass'
-    }
-    catch (e) {
-      failTask(test.result, e)
-    }
+        if (runner.runTest) {
+          await runner.runTest(test)
+        }
+        else {
+          const fn = getFn(test)
+          if (!fn)
+            throw new Error('Test function is not found. Did you add it using `setFn`?')
+          await fn()
+        }
 
-    try {
-      await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
-      await callCleanupHooks(beforeEachCleanups)
-    }
-    catch (e) {
-      failTask(test.result, e)
-    }
+        await runner.onAfterTryTest?.(test, retryCount)
+
+        if (!test.repeats)
+          test.result.state = 'pass'
+        else if (test.repeats && retry === retryCount)
+          test.result.state = 'pass'
+      }
+      catch (e) {
+        failTask(test.result, e)
+      }
 
-    if (test.result.state === 'pass')
-      break
+      try {
+        await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
+        await callCleanupHooks(beforeEachCleanups)
+      }
+      catch (e) {
+        failTask(test.result, e)
+      }
 
-    // update retry info
-    updateTask(test, runner)
+      if (test.result.state === 'pass')
+        break
+      // update retry info
+      updateTask(test, runner)
+    }
   }
 
   if (test.result.state === 'fail')
@@ -240,68 +248,75 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
     suite.result.state = 'todo'
   }
   else {
-    try {
-      beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite])
+    const retry = suite.retry || 1
 
-      if (runner.runSuite) {
-        await runner.runSuite(suite)
-      }
-      else {
-        for (let tasksGroup of partitionSuiteChildren(suite)) {
-          if (tasksGroup[0].concurrent === true) {
-            const mutex = limit(runner.config.maxConcurrency)
-            await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner))))
-          }
-          else {
-            const { sequence } = runner.config
-            if (sequence.shuffle || suite.shuffle) {
-              // run describe block independently from tests
-              const suites = tasksGroup.filter(group => group.type === 'suite')
-              const tests = tasksGroup.filter(group => group.type === 'test')
-              const groups = shuffle([suites, tests], sequence.seed)
-              tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed))
+    for (let retryCount = 0; retryCount < retry; retryCount++) {
+      try {
+        beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite])
+
+        if (runner.runSuite) {
+          await runner.runSuite(suite)
+        }
+        else {
+          for (let tasksGroup of partitionSuiteChildren(suite)) {
+            if (tasksGroup[0].concurrent === true) {
+              const mutex = limit(runner.config.maxConcurrency)
+              await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner))))
+            }
+            else {
+              const { sequence } = runner.config
+              if (sequence.shuffle || suite.shuffle) {
+                // run describe block independently from tests
+                const suites = tasksGroup.filter(group => group.type === 'suite')
+                const tests = tasksGroup.filter(group => group.type === 'test')
+                const groups = shuffle([suites, tests], sequence.seed)
+                tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed))
+              }
+              for (const c of tasksGroup)
+                await runSuiteChild(c, runner)
             }
-            for (const c of tasksGroup)
-              await runSuiteChild(c, runner)
           }
         }
       }
-    }
-    catch (e) {
-      failTask(suite.result, e)
-    }
-
-    try {
-      await callSuiteHook(suite, suite, 'afterAll', runner, [suite])
-      await callCleanupHooks(beforeAllCleanups)
-    }
-    catch (e) {
-      failTask(suite.result, e)
-    }
-  }
+      catch (e) {
+        failTask(suite.result, e)
+      }
 
-  suite.result.duration = now() - start
+      try {
+        await callSuiteHook(suite, suite, 'afterAll', runner, [suite])
+        await callCleanupHooks(beforeAllCleanups)
+      }
+      catch (e) {
+        failTask(suite.result, e)
+      }
 
-  if (suite.mode === 'run') {
-    if (!hasTests(suite)) {
-      suite.result.state = 'fail'
-      if (!suite.result.error) {
-        const error = processError(new Error(`No test found in suite ${suite.name}`))
-        suite.result.error = error
-        suite.result.errors = [error]
+      if (suite.mode === 'run') {
+        if (!hasTests(suite)) {
+          suite.result.state = 'fail'
+          if (!suite.result.error) {
+            const error = processError(new Error(`No test found in suite ${suite.name}`))
+            suite.result.error = error
+            suite.result.errors = [error]
+          }
+        }
+        else if (hasFailed(suite)) {
+          suite.result.state = 'fail'
+        }
+        else {
+          suite.result.state = 'pass'
+        }
       }
-    }
-    else if (hasFailed(suite)) {
-      suite.result.state = 'fail'
-    }
-    else {
-      suite.result.state = 'pass'
-    }
-  }
 
-  await runner.onAfterRunSuite?.(suite)
+      updateTask(suite, runner)
 
-  updateTask(suite, runner)
+      suite.result.duration = now() - start
+
+      await runner.onAfterRunSuite?.(suite)
+
+      if (suite.result.state === 'pass')
+        break
+    }
+  }
 }
 
 async function runSuiteChild(c: Task, runner: VitestRunner) {

packages/runner/src/suite.ts

@@ -73,6 +73,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       suite: undefined!,
       fails: this.fails,
       retry: options?.retry,
+      repeats: options?.repeats,
     } as Omit<Test, 'context'> as Test
 
     if (this.concurrent || concurrent)
@@ -124,6 +125,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
   }
 
   function initSuite() {
+    if (typeof suiteOptions === 'number')
+      suiteOptions = { timeout: suiteOptions }
+
     suite = {
       id: '',
       type: 'suite',
@@ -132,6 +136,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       shuffle,
       tasks: [],
     }
+
     setHooks(suite, createSuiteHooks())
   }
 
@@ -194,14 +199,14 @@ function createSuite() {
   suiteFn.runIf = (condition: any) => (condition ? suite : suite.skip) as SuiteAPI
 
   return createChainable(
-    ['concurrent', 'shuffle', 'skip', 'only', 'todo'],
+    ['concurrent', 'shuffle', 'skip', 'only', 'todo', 'repeats'],
     suiteFn,
   ) as unknown as SuiteAPI
 }
 
 function createTest(fn: (
   (
-    this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails', boolean | undefined>,
+    this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails' | 'repeats', boolean | undefined>,
     title: string,
     fn?: TestFunction,
     options?: number | TestOptions
@@ -231,7 +236,7 @@ function createTest(fn: (
   testFn.runIf = (condition: any) => (condition ? test : test.skip) as TestAPI
 
   return createChainable(
-    ['concurrent', 'skip', 'only', 'todo', 'fails'],
+    ['concurrent', 'skip', 'only', 'todo', 'fails', 'repeats'],
     testFn,
   ) as TestAPI
 }

packages/runner/src/types/tasks.ts

@@ -16,6 +16,7 @@ export interface TaskBase {
   result?: TaskResult
   retry?: number
   meta?: any
+  repeats?: number
 }
 
 export interface TaskCustom extends TaskBase {
@@ -35,6 +36,7 @@ export interface TaskResult {
   htmlError?: string
   hooks?: Partial<Record<keyof SuiteHooks, TaskState>>
   retryCount?: number
+  repeatCount?: number
 }
 
 export type TaskResultPack = [id: string, result: TaskResult | undefined]
@@ -140,7 +142,7 @@ interface TestEachFunction {
 }
 
 type ChainableTestAPI<ExtraContext = {}> = ChainableFunction<
-  'concurrent' | 'only' | 'skip' | 'todo' | 'fails',
+  'concurrent' | 'only' | 'skip' | 'todo' | 'fails' | 'repeats',
   [name: string, fn?: TestFunction<ExtraContext>, options?: number | TestOptions],
   void,
   {
@@ -161,6 +163,12 @@ export interface TestOptions {
    * @default 1
    */
   retry?: number
+  /**
+   * How many times the test will repeat.
+   *
+   * @default 5
+   */
+  repeats?: number
 }
 
 export type TestAPI<ExtraContext = {}> = ChainableTestAPI<ExtraContext> & {
@@ -170,7 +178,7 @@ export type TestAPI<ExtraContext = {}> = ChainableTestAPI<ExtraContext> & {
 }
 
 type ChainableSuiteAPI<ExtraContext = {}> = ChainableFunction<
-  'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle',
+  'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle' | 'repeats',
   [name: string, factory?: SuiteFactory<ExtraContext>, options?: number | TestOptions],
   SuiteCollector<ExtraContext>,
   {

packages/vitest/src/node/reporters/renderers/listRenderer.ts

@@ -107,6 +107,9 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
     if (task.mode === 'skip' || task.mode === 'todo')
       suffix += ` ${c.dim(c.gray('[skipped]'))}`
 
+    if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1)
+      suffix += c.yellow(` (repeat x${task.result.repeatCount})`)
+
     if (task.result?.duration != null) {
       if (task.result.duration > DURATION_LONG)
         suffix += c.yellow(` ${Math.round(task.result.duration)}${c.dim('ms')}`)

packages/vitest/src/typecheck/collect.ts

@@ -83,7 +83,7 @@ export async function collectTests(ctx: Vitest, filepath: string): Promise<null
       const { arguments: [{ value: message }] } = node as any
       const property = callee?.property?.name
       let mode = !property || property === name ? 'run' : property
-      if (!['run', 'skip', 'todo', 'only', 'skipIf', 'runIf'].includes(mode))
+      if (!['run', 'skip', 'todo', 'only', 'skipIf', 'runIf', 'repeats'].includes(mode))
         throw new Error(`${name}.${mode} syntax is not supported when testing types`)
       // cannot statically analyze, so we always skip it
       if (mode === 'skipIf' || mode === 'runIf')