Blocks E2E: Refactor configs and workflow

.github/workflows/blocks-playwright.yml

@@ -1,4 +1,4 @@
-name: Run Blocks Playwright Tests
+name: Blocks Playwright Tests
 
 on:
     pull_request:
@@ -11,13 +11,12 @@ on:
     # Allow manually triggering the workflow.
     workflow_dispatch:
 
-
 env:
     FORCE_COLOR: 1
 
 jobs:
-    e2e:
-        name: ${{ matrix.config.name }} [${{ matrix.shards.name }}]
+    blocks-playwright-tests:
+        name: Shard ${{ matrix.shardIndex }} of ${{ matrix.shardTotal }}
         timeout-minutes: 60
         runs-on: ubuntu-latest
         defaults:
@@ -26,47 +25,48 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                config:
-                    - name: Default (Block) Theme
-                      file: playwright.config.ts
-                      resultPath: test-results
-                    - name: Classic Theme
-                      file: playwright.classic-theme.config.ts
-                      resultPath: test-results-classic-theme
-                    - name: Side Effects
-                      file: playwright.side-effects.config.ts
-                      resultPath: test-results-side-effects
-                    - name: Block Theme With Templates
-                      file: playwright.block-theme-with-templates.config.ts
-                      resultPath: test-results-block-theme-with-templates
-                shards:
-                    - name: 1/5
-                    - name: 2/5
-                    - name: 3/5
-                    - name: 4/5
-                    - name: 5/5
+                shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+                shardTotal: [10]
+
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
 
             - name: Setup WooCommerce Monorepo
               uses: ./.github/actions/setup-woocommerce-monorepo
               with:
                   install: '@woocommerce/plugin-woocommerce...'
                   build: '@woocommerce/plugin-woocommerce'
 
-            - name: Install Playwright
-              run: pnpm --filter='@woocommerce/block-library' exec playwright install --with-deps
+            - name: Install Playwright dependencies
+              run: pnpm exec playwright install chromium --with-deps
 
-            - name: Start wp-env
-              run: pnpm --filter='@woocommerce/block-library' env:start
+            - name: Setup testing environment and start the server
+              run: pnpm env:start
 
             - name: Run Playwright tests
-              working-directory: plugins/woocommerce-blocks
-              run: pnpm playwright test --config=tests/e2e/${{ matrix.config.file }} --shard ${{ matrix.shards.name }}
+              run: pnpm test:e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+            - name: Archive debug artifacts (screenshots, traces)
+              uses: actions/upload-artifact@v4
+              if: ${{ !cancelled() }}
+              with:
+                  name: failures-artifacts-shard-${{ matrix.shardIndex }}
+                  path: plugins/woocommerce-blocks/tests/e2e/artifacts/test-results
+                  if-no-files-found: ignore
 
-            - uses: actions/upload-artifact@v3
-              if: ${{ failure() }}
+    merge-artifacts:
+        # Merges all artifacts from all shards into a single zip and
+        # deletes the parts. In case of a rerun, artifacts from the
+        # previous run will be retained by merging them with the new ones. 
+        name: Merge Artifacts
+        if: ${{ !cancelled() }}
+        needs: [blocks-playwright-tests]
+        runs-on: ubuntu-latest
+        steps:
+            - name: Merge Artifacts
+              uses: actions/upload-artifact/merge@v4
+              # Don't fail the job if there aren't any artifacts to merge.
+              continue-on-error: true
               with:
-                  name: playwright-report-${{ matrix.config.name }}
-                  path: plugins/woocommerce-blocks/tests/e2e/artifacts/${{ matrix.config.resultPath }}
-                  if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
+                  name: failures-artifacts
+                  delete-merged: true

plugins/woocommerce-blocks/package.json

@@ -86,13 +86,9 @@
 		"test:js": "wp-scripts test-unit-js --config tests/js/jest.config.json",
 		"test:debug": "ndb .",
 		"test:e2e": "sh ./bin/check-env.sh && pnpm playwright test --config=tests/e2e/playwright.config.ts",
-		"test:e2e:report": "sh ./bin/check-env.sh && npx playwright test --config=tests/e2e/playwright.config.ts --reporter=html",
-		"test:e2e:side-effects": "pnpm run test:e2e --config=tests/e2e/playwright.side-effects.config.ts",
-		"test:e2e:side-effects:report": "pnpm run test:e2e:report --config=tests/e2e/playwright.side-effects.config.ts --reporter=html",
-		"test:e2e:classic-theme": "pnpm run test:e2e --config=tests/e2e/playwright.classic-theme.config.ts",
-		"test:e2e:classic-theme:report": "pnpm run test:e2e:report --config=tests/e2e/playwright.classic-theme.config.ts",
-		"test:e2e:block-theme-with-templates": "pnpm run test:e2e --config=tests/e2e/playwright.block-theme-with-templates.config.ts",
-		"test:e2e:block-theme-with-templates:report": "pnpm run test:e2e:report --config=tests/e2e/playwright.block-theme-with-templates.config.ts",
+		"test:e2e:block-theme": "pnpm run test:e2e block_theme",
+		"test:e2e:classic-theme": "pnpm run test:e2e classic_theme",
+		"test:e2e:block-theme-with-templates": "pnpm run test:e2e block_theme_with_templates",
 		"test:e2e:jest": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js",
 		"test:e2e:jest:dev": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config-dev.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js",
 		"test:e2e:jest:dev-watch": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config-dev.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js --watch",

plugins/woocommerce-blocks/tests/e2e/global-setup.ts

@@ -5,7 +5,14 @@
  */
 import { chromium, request } from '@playwright/test';
 import { RequestUtils } from '@wordpress/e2e-test-utils-playwright';
-import { BASE_URL, adminFile, cli, customerFile } from '@woocommerce/e2e-utils';
+import {
+	BASE_URL,
+	adminFile,
+	cli,
+	customerFile,
+	BLOCK_THEME_SLUG,
+	DB_EXPORT_FILE,
+} from '@woocommerce/e2e-utils';
 
 /**
  * Internal dependencies
@@ -53,36 +60,51 @@ const prepareAttributes = async () => {
 };
 
 async function globalSetup() {
-	const timers = {
-		total: '└ Total time',
-		authentication: '├ Authentication time',
-		attributes: '├ Attributes preparation time',
-	};
+	console.log( 'Running global setup:' );
+	console.time( '└ Total time' );
 
-	console.log( 'Running global setup...' );
-	console.time( timers.total );
+	let databaseImported = false;
+
+	try {
+		console.log( '├ Attempting database import…' );
+		await cli(
+			`npm run wp-env run tests-cli wp db import ${ DB_EXPORT_FILE }`
+		);
+		databaseImported = true;
+	} catch ( _error ) {
+		// noop
+	}
 
 	const requestContext = await request.newContext( {
 		baseURL: BASE_URL,
 	} );
 
-	console.time( timers.authentication );
-	await new RequestUtils( requestContext, {
-		user: admin,
-		storageStatePath: adminFile,
-	} ).setupRest();
+	console.log( '├ Pre-authenticating users…' );
 	await new RequestUtils( requestContext, {
 		user: customer,
 		storageStatePath: customerFile,
 	} ).setupRest();
-	console.timeEnd( timers.authentication );
+	const requestUtils = new RequestUtils( requestContext, {
+		user: admin,
+		storageStatePath: adminFile,
+	} );
+	await requestUtils.setupRest();
+
+	if ( ! databaseImported ) {
+		console.log( '├ Activating default theme…' );
+		await requestUtils.activateTheme( BLOCK_THEME_SLUG );
+
+		console.log( '├ Preparing product attributes…' );
+		await prepareAttributes();
+	}
 
-	console.time( timers.attributes );
-	await prepareAttributes();
-	console.timeEnd( timers.attributes );
+	console.log( '├ Exporting database…' );
+	await cli(
+		`npm run wp-env run tests-cli wp db export ${ DB_EXPORT_FILE }`
+	);
 
 	await requestContext.dispose();
-	console.timeEnd( timers.total );
+	console.timeEnd( '└ Total time' );
 }
 
 export default globalSetup;

plugins/woocommerce-blocks/tests/e2e/playwright-utils/test.ts

@@ -161,12 +161,9 @@ const test = base.extend<
 			window.localStorage.clear();
 		} );
 
-		const cliOutput = await cli(
+		await cli(
 			`npm run wp-env run tests-cli wp db import ${ DB_EXPORT_FILE }`
 		);
-		if ( ! cliOutput.stdout.includes( 'Success: Imported ' ) ) {
-			throw new Error( `Failed to import ${ DB_EXPORT_FILE }` );
-		}
 	},
 	pageUtils: async ( { page }, use ) => {
 		await use( new PageUtils( { page } ) );