[pull] main from vitest-dev:main

docs/.vitepress/config.ts

@@ -507,6 +507,10 @@ export default ({ mode }: { mode: string }) => {
                 text: 'retry',
                 link: '/config/retry',
               },
+              {
+                text: 'repeats',
+                link: '/config/repeats',
+              },
               {
                 text: 'onConsoleLog',
                 link: '/config/onconsolelog',

docs/config/repeats.md

@@ -0,0 +1,14 @@
+---
+title: repeats | Config
+outline: deep
+---
+
+# repeats
+
+- **Type:** `number`
+- **Default:** `0`
+- **CLI:** `--repeats=<number>`
+
+Repeat every test a specific number of times regardless of the result. A test that uses the [`repeats`](/api/test#repeats) test option takes precedence over this value.
+
+This is useful for verifying that tests are stable across multiple runs. If a test fails on any repetition, the whole test is reported as failed.

docs/guide/cli-generated.md

@@ -637,6 +637,13 @@ Delay in milliseconds between retry attempts (default: `0`)
 
 Regex pattern to match error messages that should trigger a retry. Only errors matching this pattern will cause a retry (default: retry on all errors)
 
+### repeats
+
+- **CLI:** `--repeats <number>`
+- **Config:** [repeats](/config/repeats)
+
+Repeat every test a specific number of times regardless of the result (default: `0`)
+
 ### diff.aAnnotation
 
 - **CLI:** `--diff.aAnnotation <annotation>`

packages/browser/src/client/channel.ts

@@ -21,6 +21,11 @@ export interface IframeViewportDoneEvent {
   iframeId: string
 }
 
+export interface IframeReadyEvent {
+  event: 'ready'
+  iframeId: string
+}
+
 export interface GlobalChannelTestRunCanceledEvent {
   type: 'cancel'
   reason: CancelReason
@@ -52,6 +57,7 @@ export type GlobalChannelIncomingEvent = GlobalChannelTestRunCanceledEvent
 
 export type IframeChannelIncomingEvent
   = | IframeViewportEvent
+    | IframeReadyEvent
 
 export type IframeChannelOutgoingEvent
   = | IframeExecuteEvent

packages/browser/src/client/orchestrator.ts

@@ -1,5 +1,5 @@
 import type { Context as OTELContext } from '@opentelemetry/api'
-import type { GlobalChannelIncomingEvent, IframeChannelIncomingEvent, IframeChannelOutgoingEvent, IframeViewportDoneEvent, IframeViewportFailEvent } from '@vitest/browser/client'
+import type { GlobalChannelIncomingEvent, IframeChannelEvent, IframeChannelOutgoingEvent, IframeViewportDoneEvent, IframeViewportFailEvent } from '@vitest/browser/client'
 import type { BrowserTesterOptions, SerializedConfig } from 'vitest'
 import type { FileSpecification } from 'vitest/internal/browser'
 import { channel, client, globalChannel } from '@vitest/browser/client'
@@ -16,6 +16,8 @@ export class IframeOrchestrator {
   private cancelled = false
   private recreateNonIsolatedIframe = false
   private iframes = new Map<string, HTMLIFrameElement>()
+  private readyIframes = new Set<string>()
+  private readyWaiters = new Map<string, () => void>()
 
   public eventTarget: EventTarget = new EventTarget()
 
@@ -91,6 +93,8 @@ export class IframeOrchestrator {
 
     this.iframes.forEach(iframe => iframe.remove())
     this.iframes.clear()
+    this.readyIframes.clear()
+    this.readyWaiters.clear()
 
     for (let i = 0; i < options.files.length; i++) {
       if (this.cancelled) {
@@ -149,8 +153,7 @@ export class IframeOrchestrator {
       // because we called "cleanup" in the previous run
       // the iframe is not removed immediately to let the user see the last test
       this.recreateNonIsolatedIframe = false
-      this.iframes.get(ID_ALL)!.remove()
-      this.iframes.delete(ID_ALL)
+      this.removeIframe(ID_ALL)
       debug('recreate non-isolated iframe')
     }
 
@@ -191,8 +194,7 @@ export class IframeOrchestrator {
     const file = spec.filepath
 
     if (this.iframes.has(file)) {
-      this.iframes.get(file)!.remove()
-      this.iframes.delete(file)
+      this.removeIframe(file)
     }
 
     await this.prepareIframe(
@@ -257,12 +259,14 @@ export class IframeOrchestrator {
         }
         else {
           this.iframes.set(iframeId, iframe)
-          this.sendEventToIframe({
-            event: 'prepare',
-            iframeId,
-            startTime,
-            otelCarrier: this.traces.getContextCarrier(otelContext),
-          }).then(resolve, error => reject(this.dispatchIframeError(error)))
+          this.waitForReady(iframeId)
+            .then(() => this.sendEventToIframe({
+              event: 'prepare',
+              iframeId,
+              startTime,
+              otelCarrier: this.traces.getContextCarrier(otelContext),
+            }))
+            .then(resolve, error => reject(this.dispatchIframeError(error)))
         }
       }
       iframe.onerror = (e) => {
@@ -280,6 +284,34 @@ export class IframeOrchestrator {
     return iframe
   }
 
+  private markReady(iframeId: string) {
+    this.readyIframes.add(iframeId)
+
+    const waiter = this.readyWaiters.get(iframeId)
+    if (waiter) {
+      this.readyWaiters.delete(iframeId)
+      waiter()
+    }
+  }
+
+  private waitForReady(iframeId: string): Promise<void> {
+    if (this.readyIframes.has(iframeId)) {
+      return Promise.resolve()
+    }
+
+    return new Promise((resolve) => {
+      this.readyWaiters.set(iframeId, resolve)
+    })
+  }
+
+  private removeIframe(iframeId: string) {
+    const iframe = this.iframes.get(iframeId)
+    this.iframes.delete(iframeId)
+    this.readyIframes.delete(iframeId)
+    this.readyWaiters.delete(iframeId)
+    iframe?.remove()
+  }
+
   private loggedIframe = new WeakSet<HTMLIFrameElement>()
 
   private createWarningMessage(iframeId: string, location: string) {
@@ -369,9 +401,13 @@ export class IframeOrchestrator {
     }
   }
 
-  private async onIframeEvent(e: MessageEvent<IframeChannelIncomingEvent>) {
+  private async onIframeEvent(e: MessageEvent<IframeChannelEvent>) {
     debug('iframe event', JSON.stringify(e.data))
     switch (e.data.event) {
+      case 'ready': {
+        this.markReady(e.data.iframeId)
+        break
+      }
       case 'viewport': {
         const { width, height, iframeId: id } = e.data
         const iframe = this.iframes.get(id)

packages/browser/src/client/tester/tester.ts

@@ -118,6 +118,11 @@ getBrowserState().iframeId = iframeId
 
 registerPageMarkHandler((name, options) => page.mark(name, options))
 
+channel.postMessage({
+  event: 'ready',
+  iframeId,
+})
+
 let contextSwitched = false
 
 async function prepareTestEnvironment(options: PrepareOptions) {

packages/vitest/src/node/cli/cli-config.ts

@@ -630,6 +630,11 @@ export const cliOptionsConfig: VitestCLIOptions = {
       },
     },
   },
+  repeats: {
+    description:
+      'Repeat every test a specific number of times regardless of the result (default: `0`)',
+    argument: '<number>',
+  },
   diff: {
     description:
       'DiffOptions object or a path to a module which exports DiffOptions object',

packages/vitest/src/node/config/serializeConfig.ts

@@ -45,6 +45,7 @@ export function serializeConfig(project: TestProject): SerializedConfig {
     // TODO: non serializable function?
     diff: config.diff,
     retry: config.retry,
+    repeats: config.repeats,
     disableConsoleIntercept: config.disableConsoleIntercept,
     root: config.root,
     name: config.name,

packages/vitest/src/node/projects/resolveProjects.ts

@@ -52,6 +52,7 @@ export async function resolveProjects(
     'expandSnapshotDiff',
     'disableConsoleIntercept',
     'retry',
+    'repeats',
     'testNamePattern',
     'passWithNoTests',
     'bail',

packages/vitest/src/node/types/config.ts

@@ -830,6 +830,13 @@ export interface InlineConfig {
    */
   retry?: SerializableRetry
 
+  /**
+   * Repeat every test a specific number of times regardless of the result.
+   *
+   * @default 0 // Don't repeat
+   */
+  repeats?: number
+
   /**
    * Show full diff when snapshot fails instead of a patch.
    */

packages/vitest/src/runtime/config.ts

@@ -31,6 +31,7 @@ export interface SerializedConfig {
   testTimeout: number
   hookTimeout: number
   retry: SerializableRetry
+  repeats?: number
   includeTaskLocation: boolean | undefined
   tags: TestTagDefinition[]
   tagsFilter: string[] | undefined

packages/vitest/src/runtime/runner/suite.ts

@@ -387,7 +387,7 @@ function createSuiteCollector(
       file: (currentSuite?.file ?? collectorContext.currentSuite?.file)!,
       timeout,
       retry: options.retry ?? runner.config.retry,
-      repeats: options.repeats,
+      repeats: options.repeats ?? runner.config.repeats,
       mode: options.only
         ? 'only'
         : options.skip

test/browser/specs/readiness.test.ts

@@ -0,0 +1,60 @@
+import { expect, test } from 'vitest'
+import { instances, runInlineBrowserTests } from './utils'
+
+test('prepare waits until the tester can receive browser channel events', { timeout: 5000 }, async () => {
+  const { stderr, testTree } = await runInlineBrowserTests(
+    {
+      'basic.test.ts': `
+        import { expect, test } from 'vitest'
+
+        test('runs in the browser', () => {
+          expect(1).toBe(1)
+        })
+      `,
+      'delayed-tester.html': `
+        <!DOCTYPE html>
+        <html lang="en">
+          <head>
+            <meta charset="UTF-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+            <title>Delayed Tester</title>
+            <script>
+              const addEventListener = BroadcastChannel.prototype.addEventListener
+              const postMessage = BroadcastChannel.prototype.postMessage
+              BroadcastChannel.prototype.addEventListener = function(type, listener, options) {
+                if (type === 'message') {
+                  setTimeout(() => addEventListener.call(this, type, listener, options), 100)
+                  return
+                }
+                return addEventListener.call(this, type, listener, options)
+              }
+              BroadcastChannel.prototype.postMessage = function(message) {
+                if (message && message.event === 'ready') {
+                  setTimeout(() => postMessage.call(this, message), 150)
+                  return
+                }
+                return postMessage.call(this, message)
+              }
+            </script>
+          </head>
+          <body></body>
+        </html>
+      `,
+    },
+    {
+      browser: {
+        instances: [instances[0]],
+        testerHtmlPath: './delayed-tester.html',
+      },
+    },
+  )
+
+  expect(stderr).toBe('')
+  expect(testTree()).toMatchInlineSnapshot(`
+    {
+      "basic.test.ts": {
+        "runs in the browser": "passed",
+      },
+    }
+  `)
+})

test/e2e/fixtures/repeats/repeats-config.test.ts

@@ -0,0 +1,9 @@
+import { expect, test } from 'vitest'
+
+test('uses repeats from config', () => {
+  expect(1 + 1).toBe(2)
+})
+
+test('test option overrides config', { repeats: 1 }, () => {
+  expect(1 + 1).toBe(2)
+})

test/e2e/test/cli-config.test.ts

@@ -27,6 +27,7 @@ it('correctly inherit from the cli', async () => {
       globals: true,
       expandSnapshotDiff: true,
       retry: 6,
+      repeats: 3,
       testNamePattern: 'math',
       passWithNoTests: true,
       bail: 100,
@@ -50,6 +51,7 @@ it('correctly inherit from the cli', async () => {
     globals: true,
     expandSnapshotDiff: true,
     retry: 6,
+    repeats: 3,
     passWithNoTests: true,
     bail: 100,
     experimental: {

test/e2e/test/repeats.test.ts

@@ -0,0 +1,26 @@
+import type { TestModule } from 'vitest/node'
+import { resolve } from 'pathe'
+import { expect, test } from 'vitest'
+import { runVitest } from '../../test-utils'
+
+const root = resolve(__dirname, '..', 'fixtures', 'repeats')
+
+test('repeats config option is exposed to tests and repeats execution', async () => {
+  const { ctx } = await runVitest({
+    root,
+    include: ['repeats-config.test.ts'],
+    repeats: 3,
+  })
+
+  const file = ctx!.state.getFiles()[0]
+  const testModule = ctx!.state.getReportedEntity(file)! as TestModule
+  const tests = [...testModule.children.allTests()]
+
+  const fromConfig = tests.find(t => t.name === 'uses repeats from config')!
+  expect(fromConfig.options.repeats).toBe(3)
+  expect(fromConfig.diagnostic()!.repeatCount).toBe(3)
+
+  const overridden = tests.find(t => t.name === 'test option overrides config')!
+  expect(overridden.options.repeats).toBe(1)
+  expect(overridden.diagnostic()!.repeatCount).toBe(1)
+})