diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 6a57443d8011..c42839e96c83 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -507,6 +507,10 @@ export default ({ mode }: { mode: string }) => { text: 'retry', link: '/config/retry', }, + { + text: 'repeats', + link: '/config/repeats', + }, { text: 'onConsoleLog', link: '/config/onconsolelog', diff --git a/docs/config/repeats.md b/docs/config/repeats.md new file mode 100644 index 000000000000..733644c5fc0d --- /dev/null +++ b/docs/config/repeats.md @@ -0,0 +1,14 @@ +--- +title: repeats | Config +outline: deep +--- + +# repeats + +- **Type:** `number` +- **Default:** `0` +- **CLI:** `--repeats=` + +Repeat every test a specific number of times regardless of the result. A test that uses the [`repeats`](/api/test#repeats) test option takes precedence over this value. + +This is useful for verifying that tests are stable across multiple runs. If a test fails on any repetition, the whole test is reported as failed. diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 7538e4f2e72f..9fe52871307f 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -637,6 +637,13 @@ Delay in milliseconds between retry attempts (default: `0`) Regex pattern to match error messages that should trigger a retry. Only errors matching this pattern will cause a retry (default: retry on all errors) +### repeats + +- **CLI:** `--repeats ` +- **Config:** [repeats](/config/repeats) + +Repeat every test a specific number of times regardless of the result (default: `0`) + ### diff.aAnnotation - **CLI:** `--diff.aAnnotation ` diff --git a/packages/browser/src/client/channel.ts b/packages/browser/src/client/channel.ts index 9dc9b33abfa9..1a88b29332e2 100644 --- a/packages/browser/src/client/channel.ts +++ b/packages/browser/src/client/channel.ts @@ -21,6 +21,11 @@ export interface IframeViewportDoneEvent { iframeId: string } +export interface IframeReadyEvent { + event: 'ready' + iframeId: string +} + export interface GlobalChannelTestRunCanceledEvent { type: 'cancel' reason: CancelReason @@ -52,6 +57,7 @@ export type GlobalChannelIncomingEvent = GlobalChannelTestRunCanceledEvent export type IframeChannelIncomingEvent = | IframeViewportEvent + | IframeReadyEvent export type IframeChannelOutgoingEvent = | IframeExecuteEvent diff --git a/packages/browser/src/client/orchestrator.ts b/packages/browser/src/client/orchestrator.ts index 48a76e82416a..29cba9927044 100644 --- a/packages/browser/src/client/orchestrator.ts +++ b/packages/browser/src/client/orchestrator.ts @@ -1,5 +1,5 @@ import type { Context as OTELContext } from '@opentelemetry/api' -import type { GlobalChannelIncomingEvent, IframeChannelIncomingEvent, IframeChannelOutgoingEvent, IframeViewportDoneEvent, IframeViewportFailEvent } from '@vitest/browser/client' +import type { GlobalChannelIncomingEvent, IframeChannelEvent, IframeChannelOutgoingEvent, IframeViewportDoneEvent, IframeViewportFailEvent } from '@vitest/browser/client' import type { BrowserTesterOptions, SerializedConfig } from 'vitest' import type { FileSpecification } from 'vitest/internal/browser' import { channel, client, globalChannel } from '@vitest/browser/client' @@ -16,6 +16,8 @@ export class IframeOrchestrator { private cancelled = false private recreateNonIsolatedIframe = false private iframes = new Map() + private readyIframes = new Set() + private readyWaiters = new Map void>() public eventTarget: EventTarget = new EventTarget() @@ -91,6 +93,8 @@ export class IframeOrchestrator { this.iframes.forEach(iframe => iframe.remove()) this.iframes.clear() + this.readyIframes.clear() + this.readyWaiters.clear() for (let i = 0; i < options.files.length; i++) { if (this.cancelled) { @@ -149,8 +153,7 @@ export class IframeOrchestrator { // because we called "cleanup" in the previous run // the iframe is not removed immediately to let the user see the last test this.recreateNonIsolatedIframe = false - this.iframes.get(ID_ALL)!.remove() - this.iframes.delete(ID_ALL) + this.removeIframe(ID_ALL) debug('recreate non-isolated iframe') } @@ -191,8 +194,7 @@ export class IframeOrchestrator { const file = spec.filepath if (this.iframes.has(file)) { - this.iframes.get(file)!.remove() - this.iframes.delete(file) + this.removeIframe(file) } await this.prepareIframe( @@ -257,12 +259,14 @@ export class IframeOrchestrator { } else { this.iframes.set(iframeId, iframe) - this.sendEventToIframe({ - event: 'prepare', - iframeId, - startTime, - otelCarrier: this.traces.getContextCarrier(otelContext), - }).then(resolve, error => reject(this.dispatchIframeError(error))) + this.waitForReady(iframeId) + .then(() => this.sendEventToIframe({ + event: 'prepare', + iframeId, + startTime, + otelCarrier: this.traces.getContextCarrier(otelContext), + })) + .then(resolve, error => reject(this.dispatchIframeError(error))) } } iframe.onerror = (e) => { @@ -280,6 +284,34 @@ export class IframeOrchestrator { return iframe } + private markReady(iframeId: string) { + this.readyIframes.add(iframeId) + + const waiter = this.readyWaiters.get(iframeId) + if (waiter) { + this.readyWaiters.delete(iframeId) + waiter() + } + } + + private waitForReady(iframeId: string): Promise { + if (this.readyIframes.has(iframeId)) { + return Promise.resolve() + } + + return new Promise((resolve) => { + this.readyWaiters.set(iframeId, resolve) + }) + } + + private removeIframe(iframeId: string) { + const iframe = this.iframes.get(iframeId) + this.iframes.delete(iframeId) + this.readyIframes.delete(iframeId) + this.readyWaiters.delete(iframeId) + iframe?.remove() + } + private loggedIframe = new WeakSet() private createWarningMessage(iframeId: string, location: string) { @@ -369,9 +401,13 @@ export class IframeOrchestrator { } } - private async onIframeEvent(e: MessageEvent) { + private async onIframeEvent(e: MessageEvent) { debug('iframe event', JSON.stringify(e.data)) switch (e.data.event) { + case 'ready': { + this.markReady(e.data.iframeId) + break + } case 'viewport': { const { width, height, iframeId: id } = e.data const iframe = this.iframes.get(id) diff --git a/packages/browser/src/client/tester/tester.ts b/packages/browser/src/client/tester/tester.ts index 5c1f559fb318..7a1511ce41c0 100644 --- a/packages/browser/src/client/tester/tester.ts +++ b/packages/browser/src/client/tester/tester.ts @@ -118,6 +118,11 @@ getBrowserState().iframeId = iframeId registerPageMarkHandler((name, options) => page.mark(name, options)) +channel.postMessage({ + event: 'ready', + iframeId, +}) + let contextSwitched = false async function prepareTestEnvironment(options: PrepareOptions) { diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index e41c744d274a..1348bea2ad05 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -630,6 +630,11 @@ export const cliOptionsConfig: VitestCLIOptions = { }, }, }, + repeats: { + description: + 'Repeat every test a specific number of times regardless of the result (default: `0`)', + argument: '', + }, diff: { description: 'DiffOptions object or a path to a module which exports DiffOptions object', diff --git a/packages/vitest/src/node/config/serializeConfig.ts b/packages/vitest/src/node/config/serializeConfig.ts index 0ce56e9ada6e..b09e7fb39929 100644 --- a/packages/vitest/src/node/config/serializeConfig.ts +++ b/packages/vitest/src/node/config/serializeConfig.ts @@ -45,6 +45,7 @@ export function serializeConfig(project: TestProject): SerializedConfig { // TODO: non serializable function? diff: config.diff, retry: config.retry, + repeats: config.repeats, disableConsoleIntercept: config.disableConsoleIntercept, root: config.root, name: config.name, diff --git a/packages/vitest/src/node/projects/resolveProjects.ts b/packages/vitest/src/node/projects/resolveProjects.ts index c739917ff360..dccb6d191d2a 100644 --- a/packages/vitest/src/node/projects/resolveProjects.ts +++ b/packages/vitest/src/node/projects/resolveProjects.ts @@ -52,6 +52,7 @@ export async function resolveProjects( 'expandSnapshotDiff', 'disableConsoleIntercept', 'retry', + 'repeats', 'testNamePattern', 'passWithNoTests', 'bail', diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index 4f96e1092f4e..307279a1804b 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -830,6 +830,13 @@ export interface InlineConfig { */ retry?: SerializableRetry + /** + * Repeat every test a specific number of times regardless of the result. + * + * @default 0 // Don't repeat + */ + repeats?: number + /** * Show full diff when snapshot fails instead of a patch. */ diff --git a/packages/vitest/src/runtime/config.ts b/packages/vitest/src/runtime/config.ts index 3493b21472e4..f7a348b37fd1 100644 --- a/packages/vitest/src/runtime/config.ts +++ b/packages/vitest/src/runtime/config.ts @@ -31,6 +31,7 @@ export interface SerializedConfig { testTimeout: number hookTimeout: number retry: SerializableRetry + repeats?: number includeTaskLocation: boolean | undefined tags: TestTagDefinition[] tagsFilter: string[] | undefined diff --git a/packages/vitest/src/runtime/runner/suite.ts b/packages/vitest/src/runtime/runner/suite.ts index a616d46bd331..e4670f00ae76 100644 --- a/packages/vitest/src/runtime/runner/suite.ts +++ b/packages/vitest/src/runtime/runner/suite.ts @@ -387,7 +387,7 @@ function createSuiteCollector( file: (currentSuite?.file ?? collectorContext.currentSuite?.file)!, timeout, retry: options.retry ?? runner.config.retry, - repeats: options.repeats, + repeats: options.repeats ?? runner.config.repeats, mode: options.only ? 'only' : options.skip diff --git a/test/browser/specs/readiness.test.ts b/test/browser/specs/readiness.test.ts new file mode 100644 index 000000000000..511466b4ec55 --- /dev/null +++ b/test/browser/specs/readiness.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'vitest' +import { instances, runInlineBrowserTests } from './utils' + +test('prepare waits until the tester can receive browser channel events', { timeout: 5000 }, async () => { + const { stderr, testTree } = await runInlineBrowserTests( + { + 'basic.test.ts': ` + import { expect, test } from 'vitest' + + test('runs in the browser', () => { + expect(1).toBe(1) + }) + `, + 'delayed-tester.html': ` + + + + + + Delayed Tester + + + + + `, + }, + { + browser: { + instances: [instances[0]], + testerHtmlPath: './delayed-tester.html', + }, + }, + ) + + expect(stderr).toBe('') + expect(testTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "runs in the browser": "passed", + }, + } + `) +}) diff --git a/test/e2e/fixtures/repeats/repeats-config.test.ts b/test/e2e/fixtures/repeats/repeats-config.test.ts new file mode 100644 index 000000000000..a19bbcaebf46 --- /dev/null +++ b/test/e2e/fixtures/repeats/repeats-config.test.ts @@ -0,0 +1,9 @@ +import { expect, test } from 'vitest' + +test('uses repeats from config', () => { + expect(1 + 1).toBe(2) +}) + +test('test option overrides config', { repeats: 1 }, () => { + expect(1 + 1).toBe(2) +}) diff --git a/test/e2e/test/cli-config.test.ts b/test/e2e/test/cli-config.test.ts index 1f228d428497..0311ea79cc15 100644 --- a/test/e2e/test/cli-config.test.ts +++ b/test/e2e/test/cli-config.test.ts @@ -27,6 +27,7 @@ it('correctly inherit from the cli', async () => { globals: true, expandSnapshotDiff: true, retry: 6, + repeats: 3, testNamePattern: 'math', passWithNoTests: true, bail: 100, @@ -50,6 +51,7 @@ it('correctly inherit from the cli', async () => { globals: true, expandSnapshotDiff: true, retry: 6, + repeats: 3, passWithNoTests: true, bail: 100, experimental: { diff --git a/test/e2e/test/repeats.test.ts b/test/e2e/test/repeats.test.ts new file mode 100644 index 000000000000..7e11c28ecbdd --- /dev/null +++ b/test/e2e/test/repeats.test.ts @@ -0,0 +1,26 @@ +import type { TestModule } from 'vitest/node' +import { resolve } from 'pathe' +import { expect, test } from 'vitest' +import { runVitest } from '../../test-utils' + +const root = resolve(__dirname, '..', 'fixtures', 'repeats') + +test('repeats config option is exposed to tests and repeats execution', async () => { + const { ctx } = await runVitest({ + root, + include: ['repeats-config.test.ts'], + repeats: 3, + }) + + const file = ctx!.state.getFiles()[0] + const testModule = ctx!.state.getReportedEntity(file)! as TestModule + const tests = [...testModule.children.allTests()] + + const fromConfig = tests.find(t => t.name === 'uses repeats from config')! + expect(fromConfig.options.repeats).toBe(3) + expect(fromConfig.diagnostic()!.repeatCount).toBe(3) + + const overridden = tests.find(t => t.name === 'test option overrides config')! + expect(overridden.options.repeats).toBe(1) + expect(overridden.diagnostic()!.repeatCount).toBe(1) +})