feat(ui): add tabbed failure view for toMatchScreenshot with comparison slider (#8813)

Author: macarie

packages/browser/src/client/tester/expect/toMatchScreenshot.ts

@@ -1,8 +1,10 @@
 import type { AsyncExpectationResult, MatcherState } from '@vitest/expect'
+import type { VisualRegressionArtifact } from '@vitest/runner'
 import type { ScreenshotMatcherOptions } from '../../../../context'
 import type { ScreenshotMatcherArguments, ScreenshotMatcherOutput } from '../../../shared/screenshotMatcher/types'
 import type { Locator } from '../locators'
-import { getBrowserState, getWorkerState } from '../../utils'
+import { recordArtifact } from '@vitest/runner'
+import { getBrowserState } from '../../utils'
 import { convertToSelector } from '../tester-utils'
 
 const counters = new Map<string, { current: number }>([])
@@ -19,13 +21,11 @@ export default async function toMatchScreenshot(
     throw new Error('\'toMatchScreenshot\' cannot be used with "not"')
   }
 
-  const currentTest = getWorkerState().current
-
-  if (currentTest === undefined || this.currentTestName === undefined) {
+  if (this.task === undefined || this.currentTestName === undefined) {
     throw new Error('\'toMatchScreenshot\' cannot be used without test context')
   }
 
-  const counterName = `${currentTest.result?.repeatCount ?? 0}${this.testPath}${this.currentTestName}`
+  const counterName = `${this.task.result?.repeatCount ?? 0}${this.testPath}${this.currentTestName}`
   let counter = counters.get(counterName)
 
   if (counter === undefined) {
@@ -66,24 +66,29 @@ export default async function toMatchScreenshot(
     ] satisfies ScreenshotMatcherArguments,
   )
 
-  if (result.pass === false && 'context' in currentTest) {
-    const { annotate } = currentTest.context
-
-    const annotations: ReturnType<typeof annotate>[] = []
+  if (result.pass === false) {
+    const attachments: VisualRegressionArtifact['attachments'] = []
 
     if (result.reference) {
-      annotations.push(annotate('Reference screenshot', { path: result.reference }))
+      attachments.push({ name: 'reference', ...result.reference })
     }
 
     if (result.actual) {
-      annotations.push(annotate('Actual screenshot', { path: result.actual }))
+      attachments.push({ name: 'actual', ...result.actual })
     }
 
     if (result.diff) {
-      annotations.push(annotate('Diff', { path: result.diff }))
+      attachments.push({ name: 'diff', path: result.diff })
     }
 
-    await Promise.all(annotations)
+    if (attachments.length > 0) {
+      await recordArtifact(this.task, {
+        type: 'internal:toMatchScreenshot',
+        kind: 'visual-regression',
+        message: result.message,
+        attachments,
+      })
+    }
   }
 
   return {
@@ -96,14 +101,15 @@ export default async function toMatchScreenshot(
             '',
             result.message,
             result.reference
-              ? `\nReference screenshot:\n  ${this.utils.EXPECTED_COLOR(result.reference)}`
+              ? `\nReference screenshot:\n  ${this.utils.EXPECTED_COLOR(result.reference.path)}`
               : null,
             result.actual
-              ? `\nActual screenshot:\n  ${this.utils.RECEIVED_COLOR(result.actual)}`
+              ? `\nActual screenshot:\n  ${this.utils.RECEIVED_COLOR(result.actual.path)}`
               : null,
             result.diff
               ? this.utils.DIM_COLOR(`\nDiff image:\n  ${result.diff}`)
               : null,
+            '',
           ]
             .filter(element => element !== null)
             .join('\n'),

packages/browser/src/node/commands/screenshotMatcher/index.ts

@@ -55,7 +55,7 @@ export const screenshotMatcher: BrowserCommand<
   if (value === null || value.actual === null) {
     return {
       pass: false,
-      reference: referenceFile && paths.reference,
+      reference: referenceFile && { path: paths.reference, width: reference!.metadata.width, height: reference!.metadata.height },
       actual: null,
       diff: null,
       message: `Could not capture a stable screenshot within ${timeout}ms.`,
@@ -80,7 +80,8 @@ export const screenshotMatcher: BrowserCommand<
     if (updateSnapshot !== 'all') {
       return {
         pass: false,
-        reference: referencePath,
+        // we use `actual`'s metadata because that's the screenshot we saved
+        reference: { path: referencePath, width: value.actual.metadata.width, height: value.actual.metadata.height },
         actual: null,
         diff: null,
         message: `No existing reference screenshot found${
@@ -143,8 +144,8 @@ export const screenshotMatcher: BrowserCommand<
   //  - fail
   return {
     pass: false,
-    reference: paths.reference,
-    actual: paths.diffs.actual,
+    reference: { path: paths.reference, width: reference.metadata.width, height: reference.metadata.height },
+    actual: { path: paths.diffs.actual, width: value.actual.metadata.width, height: value.actual.metadata.height },
     diff: finalResult.diff && paths.diffs.diff,
     message: `Screenshot does not match the stored reference.${
       finalResult.message === null

packages/browser/src/shared/screenshotMatcher/types.ts

@@ -12,11 +12,13 @@ export type ScreenshotMatcherArguments<
     },
 ]
 
+interface ScreenshotData { path: string; width: number; height: number }
+
 export type ScreenshotMatcherOutput = Promise<
   {
     pass: false
-    reference: string | null
-    actual: string | null
+    reference: ScreenshotData | null
+    actual: ScreenshotData | null
     diff: string | null
     message: string
   }

packages/runner/src/types.ts

@@ -54,4 +54,5 @@ export type {
   TestFunction,
   TestOptions,
   Use,
+  VisualRegressionArtifact,
 } from './types/tasks'

packages/runner/src/types/tasks.ts

@@ -780,6 +780,24 @@ export interface TestAnnotationArtifact extends TestArtifactBase {
   annotation: TestAnnotation
 }
 
+type VisualRegressionArtifactAttachment = TestAttachment & ({
+  name: 'reference' | 'actual'
+  width: number
+  height: number
+} | { name: 'diff' })
+
+/**
+ * @experimental
+ *
+ * Artifact type for visual regressions.
+ */
+export interface VisualRegressionArtifact extends TestArtifactBase {
+  type: 'internal:toMatchScreenshot'
+  kind: 'visual-regression'
+  message: string
+  attachments: VisualRegressionArtifactAttachment[]
+}
+
 /**
  * @experimental
  * @advanced
@@ -861,4 +879,4 @@ export interface TestArtifactRegistry {}
  *
  * This type automatically includes all artifacts registered via {@link TestArtifactRegistry}.
  */
-export type TestArtifact = TestAnnotationArtifact | TestArtifactRegistry[keyof TestArtifactRegistry]
+export type TestArtifact = TestAnnotationArtifact | VisualRegressionArtifact | TestArtifactRegistry[keyof TestArtifactRegistry]

packages/ui/client/components/artifacts/ArtifactTemplate.vue

@@ -0,0 +1,17 @@
+<script setup lang="ts">
+import { useSlots } from 'vue'
+
+const slots = useSlots()
+</script>
+
+<template>
+  <article class="flex flex-col gap-4">
+    <h1>
+      <slot name="title" />
+    </h1>
+    <p v-if="slots.message">
+      <slot name="message" />
+    </p>
+    <slot />
+  </article>
+</template>

packages/ui/client/components/artifacts/visual-regression/SmallTabs.spec.ts

@@ -0,0 +1,94 @@
+import { faker } from '@faker-js/faker'
+import { describe, expect, it } from 'vitest'
+import { userEvent } from 'vitest/browser'
+import { defineComponent, h } from 'vue'
+import { page, render } from '~/test'
+import SmallTabs from './SmallTabs.vue'
+import SmallTabsPane from './SmallTabsPane.vue'
+
+function createSmallTabs(children: number) {
+  return defineComponent({
+    setup: () =>
+      () =>
+        h(
+          SmallTabs,
+          null,
+          {
+            default: () => Array.from({ length: children }, () => h(
+              SmallTabsPane,
+              { title: faker.lorem.word() },
+              () => faker.lorem.words(2),
+            )),
+          },
+        ),
+  })
+}
+
+describe('SmallTabs', () => {
+  it('has accessible elements', async () => {
+    render(createSmallTabs(2))
+
+    // a tablist with two elements inside
+    const tablist = page.getByRole('tablist')
+    const tabs = tablist.getByRole('tab')
+    const firstTab = tabs.first()
+    const secondTab = tabs.last()
+
+    await expect.element(tablist).toBeInTheDocument()
+    expect(tabs.all()).toHaveLength(2)
+
+    await expect.element(firstTab).toHaveAttribute('aria-selected', 'true')
+    await expect.element(secondTab).toHaveAttribute('aria-selected', 'false')
+
+    // two tab panels, with one hidden
+    const panels = page.getByRole('tabpanel', { includeHidden: true })
+    const firstPanel = panels.first()
+    const secondPanel = panels.last()
+
+    expect(panels.all()).toHaveLength(2)
+
+    await expect.element(firstPanel).not.toHaveAttribute('hidden')
+    await expect.element(secondPanel).toHaveAttribute('hidden')
+
+    // panels should be labelled by their tab button
+    await expect.element(firstPanel).toHaveAttribute(
+      'aria-labelledby',
+      firstTab.element().getAttribute('id'),
+    )
+    await expect.element(secondPanel).toHaveAttribute(
+      'aria-labelledby',
+      secondTab.element().getAttribute('id'),
+    )
+
+    await expect.element(firstTab).toHaveAttribute(
+      'aria-controls',
+      firstPanel.element().getAttribute('id'),
+    )
+    await expect.element(secondTab).toHaveAttribute(
+      'aria-controls',
+      secondPanel.element().getAttribute('id'),
+    )
+  })
+
+  it('opens one panel at a time', async () => {
+    const tabsLimit = 5
+
+    render(createSmallTabs(tabsLimit))
+
+    const tabs = page.getByRole('tablist').getByRole('tab')
+    const panels = page.getByRole('tabpanel', { includeHidden: true })
+
+    for (let tabIndex = 0; tabIndex < tabsLimit; tabIndex += 1) {
+      const activeTab = tabs.nth(tabIndex)
+      const activePanel = panels.nth(tabIndex)
+
+      await userEvent.click(activeTab)
+      await expect.element(
+        tabs.and(page.getByRole('tab', { selected: true })),
+      ).toBe(activeTab.element())
+      await expect.element(
+        page.getByRole('tabpanel'),
+      ).toBe(activePanel.element())
+    }
+  })
+})

packages/ui/client/components/artifacts/visual-regression/SmallTabs.vue

@@ -0,0 +1,62 @@
+<script setup lang="ts">
+import type { SmallTabsConfig } from '~/composables/small-tabs'
+import { provide, ref, useId } from 'vue'
+import { idFor, SMALL_TABS_CONTEXT } from '~/composables/small-tabs'
+
+const activeTab = ref<string | null>(null)
+const tabs = ref<SmallTabsConfig[]>([])
+
+const id = useId()
+
+provide(SMALL_TABS_CONTEXT, {
+  id,
+  activeTab,
+  registerTab: (tab) => {
+    if (!tabs.value.some(({ id }) => id === tab.id)) {
+      tabs.value.push(tab)
+    }
+
+    if (tabs.value.length === 1) {
+      activeTab.value = tab.id
+    }
+  },
+  unregisterTab: (tab) => {
+    const index = tabs.value.findIndex(({ id }) => id === tab.id)
+
+    if (index > -1) {
+      tabs.value.splice(index, 1)
+    }
+
+    if (activeTab.value === tab.id) {
+      activeTab.value = tabs.value[0]?.id ?? null
+    }
+  },
+})
+</script>
+
+<template>
+  <div
+    class="flex flex-col items-center gap-6"
+  >
+    <div
+      role="tablist"
+      aria-orientation="horizontal"
+      class="flex gap-4"
+    >
+      <button
+        v-for="tab in tabs"
+        :id="idFor.tab(tab.id, id)"
+        :key="tab.id"
+        role="tab"
+        :aria-selected="activeTab === tab.id"
+        :aria-controls="idFor.tabpanel(tab.id, id)"
+        type="button"
+        class="aria-[selected=true]:underline underline-offset-4"
+        @click="activeTab = tab.id"
+      >
+        {{ tab.title }}
+      </button>
+    </div>
+    <slot />
+  </div>
+</template>

packages/ui/client/components/artifacts/visual-regression/SmallTabsPane.vue

@@ -0,0 +1,39 @@
+<script setup lang="ts">
+import type { SmallTabsConfig } from '~/composables/small-tabs'
+import { computed, inject, onMounted, onUnmounted, useId } from 'vue'
+import { idFor, SMALL_TABS_CONTEXT } from '~/composables/small-tabs'
+
+interface TabPaneProps extends Omit<SmallTabsConfig, 'id'> {}
+
+const props = defineProps<TabPaneProps>()
+
+const context = inject(SMALL_TABS_CONTEXT)
+
+if (!context) {
+  throw new Error('TabPane must be used within Tabs')
+}
+
+const id = useId()
+
+const isActive = computed(() => context.activeTab.value === id)
+
+onMounted(() => {
+  context.registerTab({ ...props, id })
+})
+
+onUnmounted(() => {
+  context.unregisterTab({ ...props, id })
+})
+</script>
+
+<template>
+  <div
+    :id="idFor.tabpanel(id, context.id)"
+    role="tabpanel"
+    :aria-labelledby="idFor.tab(id, context.id)"
+    :hidden="!isActive"
+    class="max-w-full"
+  >
+    <slot />
+  </div>
+</template>