ci: cache e2e prod preparation to run once instead of per-shard, various improvements and flake fixes (#15368)

Moves `prepare-run-test-against-prod:ci` into a dedicated `e2e-prep` job
that runs once and caches the result. Previously this 2-minute step ran
for every E2E shard.

Saves ~2 minutes per shard (~140+ minutes total across all E2E jobs).

Additionally, this entails a few other test suite improvements and
flakes fixes.

All together, **this reduces the amount of time our entire CI takes to
run from > 1 hour if you factor in manual retries, to about 15
minutes.**

- about 2x faster `waitForCompilation` checking, by checking more
frequently and without waiting for the load event. This previously took
about 2 seconds in CI, often running before every single test. After
`waitForCompilation` we usually navigate to a different page, so waiting
for a full page load was a waste of time.
- Reduce test timeouts. Quicker failures => quicker retries => quicker
test suite passes. The previous config was excessive
- Introduces a shorter navigation timeout. By default, it used to be
equal to `TEST_TIMEOUT`. `TEST_TIMEOUT` is high to accommodate long e2e
tests with multiple steps, not to allow page.goto to try for 3 minutes
before it fails.
- Reduces console spam due to `waitForCompilation`
- Reduce retries of `waitForCompilation`. If the dev server fails for
some reason, we shouldn't have to wait 50 (!!) minutes until the e2e
test finally fails and is ready to be retried
- Lots of flake fixes for individual tests and test suites
- Fixes flaky behavior of checkbox component. This **fixes flakiness
EVERYWHERE any checkbox is clicked**.
- Fixes upload directory clearing and seed restoration not working
correctly if there were multiple upload dirs

Author: AlessioGr

.github/workflows/e2e.config.ts

@@ -90,7 +90,7 @@ export default createE2EConfig([
   { file: 'queues', shards: 1 },
   { file: 'sort', shards: 1 },
   { file: 'server-url', shards: 1 },
-  { file: 'trash', shards: 1 },
+  { file: 'trash', shards: 2 },
   { file: 'versions', shards: 3 },
   { file: 'uploads', shards: 3 },
 ])

.github/workflows/main.yml

@@ -251,9 +251,42 @@ jobs:
           matrix=$(node .github/workflows/e2e.config.ts)
           echo "matrix=$matrix" >> $GITHUB_OUTPUT
 
+  # Prepare prod test environment once, shared by all E2E shards
+  e2e-prep:
+    name: E2E Prep
+    runs-on: ubuntu-24.04
+    needs: [e2e-matrix]
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Node setup
+        uses: ./.github/actions/setup
+        with:
+          pnpm-run-install: false
+          pnpm-restore-cache: false
+
+      - name: Restore build
+        uses: actions/cache@v4
+        with:
+          path: ./*
+          key: ${{ github.sha }}
+
+      - name: Prepare prod test environment
+        run: pnpm prepare-run-test-against-prod:ci
+
+      - name: Cache prepared test environment
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            test/packed
+            test/node_modules
+            test/package.json
+            test/pnpm-lock.yaml
+          key: e2e-prep-${{ github.sha }}
+
   tests-e2e:
     runs-on: ubuntu-24.04
-    needs: e2e-matrix
+    needs: [e2e-matrix, e2e-prep]
     name: E2E - ${{ matrix.suite }}${{ matrix.total-shards > 1 && format(' ({0}/{1})', matrix.shard, matrix.total-shards) || '' }}
     timeout-minutes: 45
     strategy:
@@ -276,6 +309,17 @@ jobs:
           path: ./*
           key: ${{ github.sha }}
 
+      - name: Restore prepared test environment
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            test/packed
+            test/node_modules
+            test/package.json
+            test/pnpm-lock.yaml
+          key: e2e-prep-${{ github.sha }}
+          fail-on-cache-miss: true
+
       - name: Start LocalStack
         run: pnpm docker:start
         if: matrix.suite == 'plugin-cloud-storage'
@@ -333,7 +377,10 @@ jobs:
         #   However, --fully-parallel would also run all tests in parallel within each shard.
         #   To maintain serial execution for most suites, we pass --workers=1 when parallel=false.
         #   Suites with parallel=true (e.g. LexicalFullyFeatured) use the default 16 workers.
-        run: PLAYWRIGHT_JSON_OUTPUT_NAME=results_${{ matrix.suite }}_${{ matrix.shard }}.json pnpm test:e2e:prod:ci:noturbo ${{ matrix.suite }} --shard=${{ matrix.shard }}/${{ matrix.total-shards }} --fully-parallel${{ matrix.parallel == false && ' --workers=1' || '' }}
+        #
+        # Note: The e2e-prep job runs prepare-run-test-against-prod:ci once and caches the result.
+        # This job restores that cache, so we use test:e2e:prod:run which skips preparation.
+        run: PLAYWRIGHT_JSON_OUTPUT_NAME=results_${{ matrix.suite }}_${{ matrix.shard }}.json pnpm test:e2e:prod:run:noturbo ${{ matrix.suite }} --shard=${{ matrix.shard }}/${{ matrix.total-shards }} --fully-parallel${{ matrix.parallel == false && ' --workers=1' || '' }}
         env:
           PLAYWRIGHT_JSON_OUTPUT_NAME: results_${{ matrix.suite }}_${{ matrix.shard }}.json
           NEXT_TELEMETRY_DISABLED: 1
@@ -555,7 +602,7 @@ jobs:
       - name: Start PostgreSQL
         uses: CasperWA/postgresql-action@v1.2
         with:
-          postgresql version: '14' # See https://hub.docker.com/_/postgres for available versions
+          postgresql version: "14" # See https://hub.docker.com/_/postgres for available versions
           postgresql db: ${{ env.POSTGRES_DB }}
           postgresql user: ${{ env.POSTGRES_USER }}
           postgresql password: ${{ env.POSTGRES_PASSWORD }}
@@ -720,4 +767,4 @@ jobs:
         if: github.event.pull_request.head.repo.fork == false
         uses: exoego/esbuild-bundle-analyzer@v1
         with:
-          metafiles: 'packages/payload/meta_index.json,packages/payload/meta_shared.json,packages/ui/meta_client.json,packages/ui/meta_shared.json,packages/next/meta_index.json,packages/richtext-lexical/meta_client.json'
+          metafiles: "packages/payload/meta_index.json,packages/payload/meta_shared.json,packages/ui/meta_client.json,packages/ui/meta_shared.json,packages/next/meta_index.json,packages/richtext-lexical/meta_client.json"

package.json

@@ -129,9 +129,11 @@
     "test:e2e:debug": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 PWDEBUG=1 DISABLE_LOGGING=true playwright test",
     "test:e2e:headed": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 DISABLE_LOGGING=true playwright test --headed",
     "test:e2e:noturbo": "pnpm runts ./test/runE2E.ts --no-turbo",
-    "test:e2e:prod": "pnpm prepare-run-test-against-prod && pnpm runts ./test/runE2E.ts --prod",
-    "test:e2e:prod:ci": "pnpm prepare-run-test-against-prod:ci && pnpm runts ./test/runE2E.ts --prod",
-    "test:e2e:prod:ci:noturbo": "pnpm prepare-run-test-against-prod:ci && pnpm runts ./test/runE2E.ts --prod --no-turbo",
+    "test:e2e:prod": "pnpm prepare-run-test-against-prod && pnpm test:e2e:prod:run",
+    "test:e2e:prod:ci": "pnpm prepare-run-test-against-prod:ci && pnpm test:e2e:prod:run",
+    "test:e2e:prod:ci:noturbo": "pnpm prepare-run-test-against-prod:ci && pnpm test:e2e:prod:run:noturbo",
+    "test:e2e:prod:run": "pnpm runts ./test/runE2E.ts --prod",
+    "test:e2e:prod:run:noturbo": "pnpm runts ./test/runE2E.ts --prod --no-turbo",
     "test:int": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 DISABLE_LOGGING=true vitest --project int",
     "test:int:firestore": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 PAYLOAD_DATABASE=firestore DISABLE_LOGGING=true vitest --project int",
     "test:int:postgres": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 PAYLOAD_DATABASE=postgres DISABLE_LOGGING=true vitest --project int",

packages/ui/src/elements/EditMany/DrawerContent.tsx

@@ -28,7 +28,6 @@ import { useConfig } from '../../providers/Config/index.js'
 import { DocumentInfoProvider } from '../../providers/DocumentInfo/index.js'
 import { useLocale } from '../../providers/Locale/index.js'
 import { OperationContext } from '../../providers/Operation/index.js'
-import { useRouteCache } from '../../providers/RouteCache/index.js'
 import { useServerFunctions } from '../../providers/ServerFunctions/index.js'
 import { useTranslation } from '../../providers/Translation/index.js'
 import { abortAndIgnore, handleAbortRef } from '../../utilities/abortAndIgnore.js'
@@ -180,7 +179,6 @@ export const EditManyDrawerContent: React.FC<EditManyDrawerContentProps> = (prop
 
   const router = useRouter()
   const abortFormStateRef = React.useRef<AbortController>(null)
-  const { clearRouteCache } = useRouteCache()
   const collectionPermissions = permissions?.collections?.[collection.slug]
   const searchParams = useSearchParams()
 
@@ -273,12 +271,12 @@ export const EditManyDrawerContent: React.FC<EditManyDrawerContentProps> = (prop
       qs.stringify(
         {
           ...parseSearchParams(searchParams),
+          _r: Date.now(), // Cache buster to force fresh data fetch. Prevents an e2e race condition where sometimes the data is not updated.
           page: selectAll ? '1' : undefined,
         },
         { addQueryPrefix: true },
       ),
     )
-    clearRouteCache()
     closeModal(drawerSlug)
 
     if (typeof onSuccessFromProps === 'function') {

packages/ui/src/elements/Pill/index.tsx

@@ -1,7 +1,7 @@
 'use client'
 import type { ElementType, HTMLAttributes } from 'react'
 
-import React from 'react' // TODO: abstract this out to support all routers
+import React, { useEffect, useState } from 'react' // TODO: abstract this out to support all routers
 
 import { Link } from '../Link/index.js'
 
@@ -95,6 +95,14 @@ const StaticPill: React.FC<PillProps> = (props) => {
     to,
   } = props
 
+  const [isHydrated, setIsHydrated] = useState(false)
+
+  useEffect(() => {
+    setIsHydrated(true)
+  }, [])
+
+  const isButton = onClick && !to
+
   const classes = [
     baseClass,
     `${baseClass}--style-${pillStyle}`,
@@ -112,7 +120,7 @@ const StaticPill: React.FC<PillProps> = (props) => {
 
   let Element: ElementType | React.FC<RenderedTypeProps> = 'div'
 
-  if (onClick && !to) {
+  if (isButton) {
     Element = 'button'
   }
 
@@ -128,10 +136,11 @@ const StaticPill: React.FC<PillProps> = (props) => {
       aria-expanded={ariaExpanded}
       aria-label={ariaLabel}
       className={classes}
+      disabled={isButton ? !isHydrated : undefined}
       href={to || null}
       id={id}
       onClick={onClick}
-      type={Element === 'button' ? 'button' : undefined}
+      type={isButton ? 'button' : undefined}
     >
       <span className={`${baseClass}__label`}>{children}</span>
       {Boolean(icon) && <span className={`${baseClass}__icon`}>{icon}</span>}

packages/ui/src/fields/Checkbox/Input.tsx

@@ -1,7 +1,7 @@
 'use client'
 import type { StaticLabel } from 'payload'
 
-import React, { useId } from 'react'
+import React, { useEffect, useId, useState } from 'react'
 
 import { RenderCustomComponent } from '../../elements/RenderCustomComponent/index.js'
 import { FieldLabel } from '../../fields/FieldLabel/index.js'
@@ -40,11 +40,19 @@ export const CheckboxInput: React.FC<CheckboxInputProps> = ({
   localized,
   onToggle,
   partialChecked,
-  readOnly,
+  readOnly: readOnlyFromProps,
   required,
 }) => {
+  const [isHydrated, setIsHydrated] = useState(false)
   const fallbackID = useId()
   const id = idFromProps || fallbackID
+
+  useEffect(() => {
+    setIsHydrated(true)
+  }, [])
+
+  const readOnly = readOnlyFromProps || !isHydrated
+
   return (
     <div
       className={[
@@ -61,11 +69,11 @@ export const CheckboxInput: React.FC<CheckboxInputProps> = ({
         <input
           aria-label=""
           aria-labelledby={name}
-          defaultChecked={Boolean(checked)}
+          checked={Boolean(checked)}
           disabled={readOnly}
           id={id}
           name={name}
-          onInput={onToggle}
+          onChange={onToggle}
           ref={inputRef}
           required={required}
           title={name}

packages/ui/src/providers/Selection/index.tsx

@@ -105,11 +105,24 @@ export const SelectionProvider: React.FC<Props> = ({ children, docs = [], totalD
       ) {
         setSelectAll(SelectAllStatus.None)
       } else {
+        const shouldSelect = selectAll !== SelectAllStatus.Some
+
         docs.forEach(({ id, _isLocked, _userEditing }) => {
           if (!_isLocked || _userEditing?.id === user?.id) {
-            rows.set(id, selectAll !== SelectAllStatus.Some)
+            rows.set(id, shouldSelect)
           }
         })
+
+        // Set selectAll synchronously to avoid race conditions with useEffect
+        if (shouldSelect && rows.size > 0) {
+          if (rows.size === docs.length) {
+            setSelectAll(SelectAllStatus.AllInPage)
+          } else {
+            setSelectAll(SelectAllStatus.Some)
+          }
+        } else {
+          setSelectAll(SelectAllStatus.None)
+        }
       }
 
       setSelected(rows)

test/access-control/e2e.spec.ts

@@ -410,7 +410,6 @@ describe('Access Control', () => {
 
       await page.goto(restrictedUrl.list)
 
-       
       expect(errors).not.toHaveLength(0)
     })
 
@@ -704,6 +703,9 @@ describe('Access Control', () => {
       const adminUserRow = page.locator('.table tr').filter({ hasText: devUser.email })
       const nonAdminUserRow = page.locator('.table tr').filter({ hasText: nonAdminEmail })
 
+      // Wait for hydration
+      await wait(1000)
+
       // Ensure admin user cannot unlock other users
       await adminUserRow.locator('.cell-id a').click()
       await page.waitForURL(`**/collections/users/**`)
@@ -715,6 +717,9 @@ describe('Access Control', () => {
 
       await page.goto(usersUrl.list)
 
+      // Wait for hydration
+      await wait(1000)
+
       // Ensure non-admin user cannot see unlock button
       await nonAdminUserRow.locator('.cell-id a').click()
       await page.waitForURL(`**/collections/users/**`)

test/admin/e2e/document-view/e2e.spec.ts

@@ -123,6 +123,8 @@ describe('Document View', () => {
   describe('API view', () => {
     test('collection — should not show API tab when disabled in config', async () => {
       await page.goto(postsUrl.collection(noApiViewCollectionSlug))
+      // Wait for hydration
+      await wait(1000)
       await page.locator('.collection-list .table a').click()
       await expect(page.locator('.doc-tabs__tabs-container')).not.toContainText('API')
     })

test/admin/e2e/general/e2e.spec.ts

@@ -477,6 +477,8 @@ describe('General', () => {
 
     test('dashboard — should navigate to collection', async () => {
       await page.goto(postsUrl.admin)
+      // Wait for hydration - otherwise playwright clicks the card early and nothing happens
+      await wait(1000)
       const anchor = page.locator(`.card-${postsCollectionSlug} a.card__click`)
       const anchorHref = await anchor.getAttribute('href')
       await anchor.click()
@@ -617,6 +619,8 @@ describe('General', () => {
 
     test('should replace history when adding query params to the URL and not push a new entry', async () => {
       await page.goto(postsUrl.admin)
+      // Wait for hydration - otherwise playwright clicks the card early and nothing happens
+      await wait(1000)
       await page.locator('.collections__card-list .card__click').first().click()
       // flaky
       // eslint-disable-next-line playwright/no-wait-for-timeout
@@ -860,6 +864,10 @@ describe('General', () => {
     })
 
     test('should allow custom translation of locale labels', async () => {
+      await page.goto(postsUrl.account)
+      // Wait for hydration - otherwise playwright clicks the localizer early and nothing happens
+      await wait(1000)
+
       const selectOptionClass = '.popup__content .popup-button-list__button'
       const localizerButton = page.locator('.localizer .popup-button')
       const localeListItem1 = page.locator(selectOptionClass).nth(0)
@@ -1035,10 +1043,33 @@ describe('General', () => {
   })
 
   describe('progress bar', () => {
-    test('should show progress bar on page navigation', async () => {
-      await page.goto(postsUrl.admin)
+    test.fixme('should show progress bar on page navigation', async () => {
+      // TODO: This test is extremely flaky in CI. Not a surprise, the progress bar only shows if the timing is right. Need to fix this and make extra sure it passes in CI without retries.
+      // eslint-disable-next-line playwright/no-networkidle
+      await page.goto(postsUrl.admin, { waitUntil: 'networkidle' })
+      // Wait for hydration - otherwise playwright clicks the card early and nothing happens
+      await wait(1000)
+
+      // Throttle network to ensure navigation takes > 500ms so progress bar is visible
+      // Progress bar has 150ms initial delay before showing, so fast navigations won't show it
+      const client = await page.context().newCDPSession(page)
+      await client.send('Network.emulateNetworkConditions', {
+        downloadThroughput: (500 * 1024) / 8, // 500 kbps
+        latency: 400, // 400ms latency
+        offline: false,
+        uploadThroughput: (500 * 1024) / 8,
+      })
+
       await page.locator('.collections__card-list .card').first().click()
       await expect(page.locator('.progress-bar')).toBeVisible()
+
+      // Reset network conditions
+      await client.send('Network.emulateNetworkConditions', {
+        downloadThroughput: -1,
+        latency: 0,
+        offline: false,
+        uploadThroughput: -1,
+      })
     })
   })
 })