feat: benchmark

docs/config/index.md

@@ -10,7 +10,7 @@ outline: deep
 
 - Create `vitest.config.ts`, which will have the higher priority and will override the configuration from `vite.config.ts`
 - Pass `--config` option to CLI, e.g. `vitest --config ./path/to/vitest.config.ts`
-- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test` if not overridden) to conditionally apply different configuration in `vite.config.ts`
+- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test`/`benchmark` if not overridden) to conditionally apply different configuration in `vite.config.ts`
 
 To configure `vitest` itself, add `test` property in your Vite config. You'll also need to add a reference to Vitest types using a [triple slash command](https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html#-reference-types-) at the top of your config file, if you are importing `defineConfig` from `vite` itself.
 

packages/ui/client/composables/summary.ts

@@ -1,5 +1,5 @@
 import { hasFailedSnapshot } from '@vitest/ws-client'
-import type { Task, Test } from 'vitest/src'
+import type { Benchmark, Task, Test } from 'vitest/src'
 import { files, testRunState } from '~/composables/client'
 
 type Nullable<T> = T | null | undefined
@@ -52,7 +52,9 @@ function toArray<T>(array?: Nullable<Arrayable<T>>): Array<T> {
     return array
   return [array]
 }
-
-function getTests(suite: Arrayable<Task>): Test[] {
-  return toArray(suite).flatMap(s => s.type === 'test' ? [s] : s.tasks.flatMap(c => c.type === 'test' ? [c] : getTests(c)))
+function isAtomTest(s: Task): s is Test | Benchmark {
+  return (s.type === 'test' || s.type === 'benchmark')
+}
+function getTests(suite: Arrayable<Task>): (Test | Benchmark)[] {
+  return toArray(suite).flatMap(s => isAtomTest(s) ? [s] : s.tasks.flatMap(c => isAtomTest(c) ? [c] : getTests(c)))
 }

packages/vitest/package.json

@@ -103,6 +103,7 @@
     "debug": "^4.3.4",
     "local-pkg": "^0.4.2",
     "strip-literal": "^0.4.0",
+    "tinybench": "^2.1.3",
     "tinypool": "^0.2.4",
     "tinyspy": "^1.0.2",
     "vite": "^2.9.12 || ^3.0.0-0"
@@ -123,6 +124,7 @@
     "chai-subset": "^1.6.0",
     "cli-truncate": "^3.1.0",
     "diff": "^5.1.0",
+    "event-target-polyfill": "^0.0.3",
     "execa": "^6.1.0",
     "fast-glob": "^3.2.11",
     "find-up": "^6.3.0",

packages/vitest/src/defaults.ts

@@ -1,8 +1,15 @@
-import type { ResolvedCoverageOptions, UserConfig } from './types'
+import type { BenchmarkUserOptions, ResolvedCoverageOptions, UserConfig } from './types'
 
 export const defaultInclude = ['**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}']
 export const defaultExclude = ['**/node_modules/**', '**/dist/**', '**/cypress/**', '**/.{idea,git,cache,output,temp}/**']
 
+export const benchmarkConfigDefaults: Required<BenchmarkUserOptions> = {
+  include: ['**/*.{bench,benchmark}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
+  exclude: defaultExclude,
+  includeSource: [],
+  reporters: ['default'],
+}
+
 const defaultCoverageExcludes = [
   'coverage/**',
   'dist/**',

packages/vitest/src/index.ts

@@ -1,4 +1,4 @@
-export { suite, test, describe, it } from './runtime/suite'
+export { suite, test, describe, it, bench } from './runtime/suite'
 export * from './runtime/hooks'
 export * from './runtime/utils'
 

packages/vitest/src/node/cli-api.ts

@@ -3,7 +3,7 @@ import type { UserConfig as ViteUserConfig } from 'vite'
 import { EXIT_CODE_RESTART } from '../constants'
 import { CoverageProviderMap } from '../integrations/coverage'
 import { getEnvPackageName } from '../integrations/env'
-import type { UserConfig } from '../types'
+import type { UserConfig, VitestRunMode } from '../types'
 import { ensurePackageInstalled } from '../utils'
 import { createVitest } from './create'
 import { registerConsoleShortcuts } from './stdin'
@@ -15,7 +15,7 @@ export interface CliOptions extends UserConfig {
   run?: boolean
 }
 
-export async function startVitest(cliFilters: string[], options: CliOptions, viteOverrides?: ViteUserConfig) {
+export async function startVitest(mode: VitestRunMode, cliFilters: string[], options: CliOptions, viteOverrides?: ViteUserConfig) {
   process.env.TEST = 'true'
   process.env.VITEST = 'true'
   process.env.NODE_ENV ??= options.mode || 'test'
@@ -36,9 +36,9 @@ export async function startVitest(cliFilters: string[], options: CliOptions, vit
   if (typeof options.coverage === 'boolean')
     options.coverage = { enabled: options.coverage }
 
-  const ctx = await createVitest(options, viteOverrides)
+  const ctx = await createVitest(mode, options, viteOverrides)
 
-  if (ctx.config.coverage.enabled) {
+  if (mode !== 'benchmark' && ctx.config.coverage.enabled) {
     const provider = ctx.config.coverage.provider || 'c8'
     if (typeof provider === 'string') {
       const requiredPackages = CoverageProviderMap[provider]

packages/vitest/src/node/cli.ts

@@ -1,6 +1,7 @@
 import cac from 'cac'
 import c from 'picocolors'
 import { version } from '../../package.json'
+import type { VitestRunMode } from '../types'
 import type { CliOptions } from './cli-api'
 import { startVitest } from './cli-api'
 import { divider } from './reporters/renderers/utils'
@@ -60,26 +61,35 @@ cli
   .command('dev [...filters]')
   .action(start)
 
+cli
+  .command('bench [...filters]')
+  .action(benchmark)
+
 cli
   .command('[...filters]')
-  .action(start)
+  .action((filter, options) => start('test', filter, options))
 
 cli.parse()
 
 async function runRelated(relatedFiles: string[] | string, argv: CliOptions) {
   argv.related = relatedFiles
   argv.passWithNoTests ??= true
-  await start([], argv)
+  await start('test', [], argv)
 }
 
 async function run(cliFilters: string[], options: CliOptions) {
   options.run = true
-  await start(cliFilters, options)
+  await start('test', cliFilters, options)
+}
+
+async function benchmark(cliFilters: string[], options: CliOptions) {
+  console.warn(c.yellow('Benchmarking is an experimental feature. API might change in the future.'))
+  await start('benchmark', cliFilters, options)
 }
 
-async function start(cliFilters: string[], options: CliOptions) {
+async function start(mode: VitestRunMode, cliFilters: string[], options: CliOptions) {
   try {
-    if (await startVitest(cliFilters, options) === false)
+    if (await startVitest(mode, cliFilters, options) === false)
       process.exit()
   }
   catch (e) {

packages/vitest/src/node/config.ts

@@ -3,9 +3,9 @@ import { normalize, resolve } from 'pathe'
 import c from 'picocolors'
 import type { ResolvedConfig as ResolvedViteConfig } from 'vite'
 
-import type { ApiConfig, ResolvedConfig, UserConfig } from '../types'
+import type { ApiConfig, ResolvedConfig, UserConfig, VitestRunMode } from '../types'
 import { defaultPort } from '../constants'
-import { configDefaults } from '../defaults'
+import { benchmarkConfigDefaults, configDefaults } from '../defaults'
 import { toArray } from '../utils'
 import { VitestCache } from './cache'
 import { BaseSequencer } from './sequencers/BaseSequencer'
@@ -61,6 +61,7 @@ export function resolveApiConfig<Options extends ApiConfig & UserConfig>(
 }
 
 export function resolveConfig(
+  mode: VitestRunMode,
   options: UserConfig,
   viteConfig: ResolvedViteConfig,
 ): ResolvedConfig {
@@ -87,6 +88,7 @@ export function resolveConfig(
     ...configDefaults,
     ...options,
     root: viteConfig.root,
+    mode,
   } as ResolvedConfig
 
   if (viteConfig.base !== '/')
@@ -157,6 +159,18 @@ export function resolveConfig(
   if (process.env.VITEST_MIN_THREADS)
     resolved.minThreads = parseInt(process.env.VITEST_MIN_THREADS)
 
+  if (mode === 'benchmark') {
+    resolved.benchmark = {
+      ...benchmarkConfigDefaults,
+      ...resolved.benchmark,
+    }
+    // override test config
+    resolved.coverage.enabled = false
+    resolved.include = resolved.benchmark.include
+    resolved.exclude = resolved.benchmark.exclude
+    resolved.includeSource = resolved.benchmark.includeSource
+  }
+
   resolved.setupFiles = toArray(resolved.setupFiles || []).map(file =>
     normalize(
       resolveModule(file, { paths: [resolved.root] })
@@ -175,6 +189,7 @@ export function resolveConfig(
     // @ts-expect-error from CLI
     ...toArray(resolved.reporter),
   ])).filter(Boolean)
+
   if (!resolved.reporters.length)
     resolved.reporters.push('default')
 

packages/vitest/src/node/core.ts

@@ -7,13 +7,13 @@ import mm from 'micromatch'
 import c from 'picocolors'
 import { ViteNodeRunner } from 'vite-node/client'
 import { ViteNodeServer } from 'vite-node/server'
-import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig } from '../types'
+import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig, VitestRunMode } from '../types'
 import { SnapshotManager } from '../integrations/snapshot/manager'
-import { clearTimeout, deepMerge, hasFailed, noop, setTimeout, slash } from '../utils'
+import { clearTimeout, deepMerge, hasFailed, noop, setTimeout, slash, toArray } from '../utils'
 import { getCoverageProvider } from '../integrations/coverage'
 import { createPool } from './pool'
 import type { WorkerPool } from './pool'
-import { createReporters } from './reporters/utils'
+import { createBenchmarkReporters, createReporters } from './reporters/utils'
 import { StateManager } from './state'
 import { resolveConfig } from './config'
 import { Logger } from './logger'
@@ -45,7 +45,9 @@ export class Vitest {
   restartsCount = 0
   runner: ViteNodeRunner = undefined!
 
-  constructor() {
+  constructor(
+    public readonly mode: VitestRunMode,
+  ) {
     this.logger = new Logger(this)
   }
 
@@ -58,7 +60,7 @@ export class Vitest {
     this.pool?.close()
     this.pool = undefined
 
-    const resolved = resolveConfig(options, server.config)
+    const resolved = resolveConfig(this.mode, options, server.config)
 
     this.server = server
     this.config = resolved
@@ -101,7 +103,9 @@ export class Vitest {
       })
     }
 
-    this.reporters = await createReporters(resolved.reporters, this.runner)
+    this.reporters = resolved.mode === 'benchmark'
+      ? await createBenchmarkReporters(toArray(resolved.benchmark?.reporters), this.runner)
+      : await createReporters(resolved.reporters, this.runner)
 
     this.runningPromise = undefined
 
@@ -490,22 +494,24 @@ export class Vitest {
   }
 
   async globTestFiles(filters: string[] = []) {
+    const { include, exclude, includeSource } = this.config
+
     const globOptions = {
       absolute: true,
       cwd: this.config.dir || this.config.root,
-      ignore: this.config.exclude,
+      ignore: exclude,
     }
 
-    let testFiles = await fg(this.config.include, globOptions)
+    let testFiles = await fg(include, globOptions)
 
     if (filters.length && process.platform === 'win32')
       filters = filters.map(f => toNamespacedPath(f))
 
     if (filters.length)
       testFiles = testFiles.filter(i => filters.some(f => i.includes(f)))
 
-    if (this.config.includeSource) {
-      let files = await fg(this.config.includeSource, globOptions)
+    if (includeSource) {
+      let files = await fg(includeSource, globOptions)
       if (filters.length)
         files = files.filter(i => filters.some(f => i.includes(f)))
 

packages/vitest/src/node/create.ts

@@ -2,13 +2,13 @@ import { resolve } from 'pathe'
 import { createServer, mergeConfig } from 'vite'
 import type { InlineConfig as ViteInlineConfig, UserConfig as ViteUserConfig } from 'vite'
 import { findUp } from 'find-up'
-import type { UserConfig } from '../types'
+import type { UserConfig, VitestRunMode } from '../types'
 import { configFiles } from '../constants'
 import { Vitest } from './core'
 import { VitestPlugin } from './plugins'
 
-export async function createVitest(options: UserConfig, viteOverrides: ViteUserConfig = {}) {
-  const ctx = new Vitest()
+export async function createVitest(mode: VitestRunMode, options: UserConfig, viteOverrides: ViteUserConfig = {}) {
+  const ctx = new Vitest(mode)
   const root = resolve(options.root || process.cwd())
 
   const configPath = options.config
@@ -19,7 +19,7 @@ export async function createVitest(options: UserConfig, viteOverrides: ViteUserC
     logLevel: 'error',
     configFile: configPath,
     // this will make "mode" = "test" inside defineConfig
-    mode: options.mode || process.env.NODE_ENV || 'test',
+    mode: options.mode || process.env.NODE_ENV || mode,
     plugins: await VitestPlugin(options, ctx),
   }
 

packages/vitest/src/node/logger.ts

@@ -78,9 +78,9 @@ export class Logger {
       this.console.error(c.dim('watch exclude:  ') + c.yellow(config.watchExclude.join(comma)))
 
     if (config.passWithNoTests)
-      this.log('No test files found, exiting with code 0\n')
+      this.log(`No ${config.mode} files found, exiting with code 0\n`)
     else
-      this.error(c.red('\nNo test files found, exiting with code 1'))
+      this.error(c.red(`\nNo ${config.mode} files found, exiting with code 1`))
   }
 
   printBanner() {

packages/vitest/src/node/plugins/index.ts

@@ -12,7 +12,7 @@ import { MocksPlugin } from './mock'
 import { CSSEnablerPlugin } from './cssEnabler'
 import { CoverageTransform } from './coverageTransform'
 
-export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest()): Promise<VitePlugin[]> {
+export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('test')): Promise<VitePlugin[]> {
   const getRoot = () => ctx.config?.root || options.root || process.cwd()
 
   async function UIPlugin() {

packages/vitest/src/node/reporters/benchmark/index.ts

@@ -0,0 +1,7 @@
+import { VerboseReporter } from '../verbose'
+import { JsonReporter } from './json'
+export const BenchmarkReportsMap = {
+  default: VerboseReporter,
+  json: JsonReporter,
+}
+export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap