fix(coverage): custom providers to work inside worker threads

packages/vitest/src/integrations/coverage.ts

@@ -1,34 +1,44 @@
 import { importModule } from 'local-pkg'
 import type { CoverageOptions, CoverageProvider, CoverageProviderModule } from '../types'
 
-export const CoverageProviderMap = {
+type Loader = (id: string) => Promise<{ default: CoverageProviderModule }>
+
+export const CoverageProviderMap: Record<string, string> = {
   c8: '@vitest/coverage-c8',
   istanbul: '@vitest/coverage-istanbul',
 }
 
-export async function resolveCoverageProvider(provider: NonNullable<CoverageOptions['provider']>) {
-  if (typeof provider === 'string') {
-    const pkg = CoverageProviderMap[provider]
-    if (!pkg)
-      throw new Error(`Unknown coverage provider: ${provider}`)
-    return await importModule<CoverageProviderModule>(pkg)
+async function resolveCoverageProviderModule(provider: NonNullable<CoverageOptions['provider']>, loader: Loader) {
+  const builtInProviderPackage = CoverageProviderMap[provider]
+
+  if (builtInProviderPackage)
+    return await importModule<CoverageProviderModule>(builtInProviderPackage)
+
+  let customProviderModule
+  try {
+    customProviderModule = await loader(provider)
   }
-  else {
-    return provider
+  catch (error) {
+    throw new Error(`Failed to load custom CoverageProviderModule from ${provider}`, { cause: error })
   }
+
+  if (customProviderModule.default == null)
+    throw new Error(`Custom CoverageProviderModule loaded from ${provider} was not the default export`)
+
+  return customProviderModule.default
 }
 
-export async function getCoverageProvider(options?: CoverageOptions): Promise<CoverageProvider | null> {
-  if (options?.enabled && options?.provider) {
-    const { getProvider } = await resolveCoverageProvider(options.provider)
+export async function getCoverageProvider(options: CoverageOptions, loader: Loader): Promise<CoverageProvider | null> {
+  if (options.enabled && options.provider) {
+    const { getProvider } = await resolveCoverageProviderModule(options.provider, loader)
     return await getProvider()
   }
   return null
 }
 
-export async function takeCoverageInsideWorker(options: CoverageOptions) {
+export async function takeCoverageInsideWorker(options: CoverageOptions, loader: Loader) {
   if (options.enabled && options.provider) {
-    const { takeCoverage } = await resolveCoverageProvider(options.provider)
+    const { takeCoverage } = await resolveCoverageProviderModule(options.provider, loader)
     return await takeCoverage?.()
   }
 }

packages/vitest/src/node/cli-api.ts

@@ -50,9 +50,9 @@ export async function startVitest(
 
   if (mode === 'test' && ctx.config.coverage.enabled) {
     const provider = ctx.config.coverage.provider || 'c8'
-    if (typeof provider === 'string') {
-      const requiredPackages = CoverageProviderMap[provider]
+    const requiredPackages = CoverageProviderMap[provider]
 
+    if (requiredPackages) {
       if (!await ensurePackageInstalled(requiredPackages, root)) {
         process.exitCode = 1
         return ctx

packages/vitest/src/node/core.ts

@@ -125,7 +125,7 @@ export class Vitest {
   async initCoverageProvider() {
     if (this.coverageProvider !== undefined)
       return
-    this.coverageProvider = await getCoverageProvider(this.config.coverage)
+    this.coverageProvider = await getCoverageProvider(this.config.coverage, id => this.runner.executeId(id))
     if (this.coverageProvider) {
       await this.coverageProvider.initialize(this)
       this.config.coverage = this.coverageProvider.resolveOptions()

packages/vitest/src/runtime/entry.ts

@@ -56,7 +56,7 @@ async function getTestRunner(config: ResolvedConfig): Promise<VitestRunner> {
 
   const originalOnAfterRun = testRunner.onAfterRun
   testRunner.onAfterRun = async (files) => {
-    const coverage = await takeCoverageInsideWorker(config.coverage)
+    const coverage = await takeCoverageInsideWorker(config.coverage, id => import(id))
     rpc().onAfterSuiteRun({ coverage })
     await originalOnAfterRun?.call(testRunner, files)
   }

packages/vitest/src/types/coverage.ts

@@ -53,12 +53,15 @@ export type CoverageReporter =
   | 'text-summary'
   | 'text'
 
-type Provider = 'c8' | 'istanbul' | CoverageProviderModule | undefined
+type BuiltinProviders = 'c8' | 'istanbul'
+type CustomProvider = string & Omit<string, BuiltinProviders>
+type Provider = BuiltinProviders | CustomProvider | undefined
 
 export type CoverageOptions<T extends Provider = Provider> =
-  T extends CoverageProviderModule ? ({ provider: T } & BaseCoverageOptions) :
-    T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) :
-        ({ provider?: T } & CoverageC8Options)
+  T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) :
+    T extends 'c8' ? ({ provider: T } & CoverageC8Options) :
+      T extends CustomProvider ? ({ provider: T } & CustomProviderOptions) :
+          ({ provider?: T } & CoverageC8Options)
 
 /** Fields that have default values. Internally these will always be defined. */
 type FieldsWithDefaultValues =
@@ -233,3 +236,5 @@ export interface CoverageC8Options extends BaseCoverageOptions {
    */
   100?: boolean
 }
+
+export type CustomProviderOptions = BaseCoverageOptions & Record<string, any>

test/coverage-test/coverage-report-tests/__snapshots__/custom-provider.report.test.ts.snap

@@ -0,0 +1,30 @@
+// Vitest Snapshot v1
+
+exports[`custom json report 1`] = `
+{
+  "calls": [
+    "initialized with context",
+    "resolveOptions",
+    "clean with force",
+    "onBeforeFilesRun",
+    "onAfterSuiteRun with {\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"}}",
+    "onAfterSuiteRun with {\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"}}",
+    "onAfterSuiteRun with {\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"}}",
+    "reportCoverage with {\\"allTestsRun\\":true}",
+  ],
+  "transformedFiles": [
+    "<process-cwd>/src/Counter/Counter.component.ts",
+    "<process-cwd>/src/Counter/Counter.component.ts?vue&type=script&src=true&lang.ts",
+    "<process-cwd>/src/Counter/Counter.vue",
+    "<process-cwd>/src/Counter/index.ts",
+    "<process-cwd>/src/Defined.vue",
+    "<process-cwd>/src/Defined.vue?vue&type=style&index=0&scoped=fdf5cd5f&lang.css",
+    "<process-cwd>/src/Hello.vue",
+    "<process-cwd>/src/another-setup.ts",
+    "<process-cwd>/src/implicitElse.ts",
+    "<process-cwd>/src/importEnv.ts",
+    "<process-cwd>/src/index.mts",
+    "<process-cwd>/src/utils.ts",
+  ],
+}
+`;

test/coverage-test/coverage-report-tests/custom-provider.report.test.ts

@@ -0,0 +1,12 @@
+/*
+ * Custom coverage provider specific test cases
+ */
+
+import { readFileSync } from 'fs'
+import { expect, test } from 'vitest'
+
+test('custom json report', async () => {
+  const report = readFileSync('./coverage/custom-coverage-provider-report.json', 'utf-8')
+
+  expect(JSON.parse(report)).toMatchSnapshot()
+})

test/coverage-test/coverage-report-tests/utils.ts

@@ -1,3 +1,4 @@
+import { readFileSync } from 'fs'
 import { normalize } from 'pathe'
 
 interface CoverageFinalJson {
@@ -17,8 +18,7 @@ interface CoverageFinalJson {
  * Normalizes paths to keep contents consistent between OS's
  */
 export async function readCoverageJson() {
-  // @ts-expect-error -- generated file
-  const { default: jsonReport } = await import('./coverage/coverage-final.json') as CoverageFinalJson
+  const jsonReport = JSON.parse(readFileSync('./coverage/coverage-final.json', 'utf8')) as CoverageFinalJson
 
   const normalizedReport: CoverageFinalJson['default'] = {}
 
@@ -30,6 +30,6 @@ export async function readCoverageJson() {
   return normalizedReport
 }
 
-function normalizeFilename(filename: string) {
+export function normalizeFilename(filename: string) {
   return normalize(filename).replace(normalize(process.cwd()), '<process-cwd>')
 }

test/coverage-test/custom-provider.ts

@@ -0,0 +1,75 @@
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'
+import type { AfterSuiteRunMeta, CoverageProvider, CoverageProviderModule, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest'
+
+import { normalizeFilename } from './coverage-report-tests/utils'
+
+const CustomCoverageProviderModule: CoverageProviderModule = {
+  getProvider(): CoverageProvider {
+    return new CustomCoverageProvider()
+  },
+
+  takeCoverage() {
+    return { customCoverage: 'Coverage report passed from workers to main thread' }
+  },
+}
+
+/**
+ * Provider that simply keeps track of the functions that were called
+ */
+class CustomCoverageProvider implements CoverageProvider {
+  name = 'custom-coverage-provider'
+
+  options!: ResolvedCoverageOptions
+  calls: string[] = []
+  transformedFiles: Set<string> = new Set()
+
+  initialize(ctx: Vitest) {
+    this.options = ctx.config.coverage
+
+    this.calls.push(`initialized ${ctx ? 'with' : 'without'} context`)
+  }
+
+  clean(force: boolean) {
+    this.calls.push(`clean ${force ? 'with' : 'without'} force`)
+  }
+
+  onBeforeFilesRun() {
+    this.calls.push('onBeforeFilesRun')
+  }
+
+  onAfterSuiteRun(meta: AfterSuiteRunMeta) {
+    this.calls.push(`onAfterSuiteRun with ${JSON.stringify(meta)}`)
+  }
+
+  reportCoverage(reportContext?: ReportContext) {
+    this.calls.push(`reportCoverage with ${JSON.stringify(reportContext)}`)
+
+    const jsonReport = JSON.stringify({
+      calls: this.calls,
+      transformedFiles: Array.from(this.transformedFiles.values()).sort(),
+    }, null, 2)
+
+    if (existsSync('./coverage'))
+      rmSync('./coverage', { maxRetries: 10, recursive: true })
+
+    mkdirSync('./coverage')
+    writeFileSync('./coverage/custom-coverage-provider-report.json', jsonReport, 'utf-8')
+  }
+
+  onFileTransform(code: string, id: string) {
+    const filename = normalizeFilename(id)
+
+    if (/\/src\//.test(filename))
+      this.transformedFiles.add(filename)
+
+    return { code }
+  }
+
+  resolveOptions(): ResolvedCoverageOptions {
+    this.calls.push('resolveOptions')
+
+    return this.options
+  }
+}
+
+export default CustomCoverageProviderModule

test/coverage-test/package.json

@@ -2,8 +2,9 @@
   "name": "@vitest/test-coverage",
   "private": true,
   "scripts": {
-    "test": "pnpm test:c8 && pnpm test:istanbul && pnpm test:types",
+    "test": "pnpm test:c8 && pnpm test:istanbul && pnpm test:custom && pnpm test:types",
     "test:c8": "node ./testing.mjs --provider c8",
+    "test:custom": "node ./testing.mjs --provider custom-provider",
     "test:istanbul": "node ./testing.mjs --provider istanbul",
     "test:types": "vitest typecheck --run"
   },

test/coverage-test/test/configuration-options.test-d.ts

@@ -1,5 +1,4 @@
 import { assertType, test } from 'vitest'
-import type { ResolvedCoverageOptions, Vitest } from 'vitest'
 import type { defineConfig } from 'vitest/config'
 
 type NarrowToTestConfig<T> = T extends { test?: any } ? NonNullable<T['test']> : never
@@ -9,40 +8,11 @@ type Coverage = NonNullable<Configuration['coverage']>
 test('providers, built-in', () => {
   assertType<Coverage>({ provider: 'c8' })
   assertType<Coverage>({ provider: 'istanbul' })
-
-  // @ts-expect-error -- String options must be known built-in's
-  assertType<Coverage>({ provider: 'unknown-reporter' })
 })
 
 test('providers, custom', () => {
   assertType<Coverage>({
-    provider: {
-      getProvider() {
-        return {
-          name: 'custom-provider',
-          initialize(_: Vitest) {},
-          resolveOptions(): ResolvedCoverageOptions {
-            return {
-              clean: true,
-              cleanOnRerun: true,
-              enabled: true,
-              exclude: ['string'],
-              extension: ['string'],
-              reporter: ['html', 'json'],
-              reportsDirectory: 'string',
-            }
-          },
-          clean(_: boolean) {},
-          onBeforeFilesRun() {},
-          onAfterSuiteRun({ coverage: _coverage }) {},
-          reportCoverage() {},
-          onFileTransform(_code: string, _id: string, ctx) {
-            ctx.getCombinedSourcemap()
-          },
-        }
-      },
-      takeCoverage() {},
-    },
+    provider: 'custom-provider',
   })
 })
 

test/coverage-test/testing.mjs

@@ -16,9 +16,9 @@ const configs = [
   // Run tests for checking coverage report contents.
   ['coverage-report-tests', {
     include: [
-      './coverage-report-tests/generic.report.test.ts',
+      ['c8', 'istanbul'].includes(provider) && './coverage-report-tests/generic.report.test.ts',
       `./coverage-report-tests/${provider}.report.test.ts`,
-    ],
+    ].filter(Boolean),
     coverage: { enabled: false, clean: false },
   }],
 ]

test/coverage-test/vitest.config.ts

@@ -14,7 +14,7 @@ export default defineConfig({
   test: {
     watch: false,
     coverage: {
-      provider: provider as any,
+      provider,
       include: ['src/**'],
       clean: true,
       all: true,