[E2E] [QA] Optimizing playwright setup and playwright.config.ts files (…

…#4741) * adding condition to not create json file if it does exist * adding cache * adding cache * updating yml files * fixing GH action * fixing flaky test * adding randomization to tests execution * reverting * fixing flaky tests * updating other gh workflows consisting of pw test run
saleor · Apr 9, 2024 · 024f2d0 · 024f2d0
1 parent 6342659
commit 024f2d0
Show file tree

Hide file tree

Showing 10 changed files with 88 additions and 62 deletions.
diff --git a/.changeset/sixty-colts-cough.md b/.changeset/sixty-colts-cough.md
@@ -0,0 +1,11 @@
+---
+"saleor-dashboard": minor
+---
+
+# Optimizing playwright setup and playwright.config.ts files
+
+-Adding a conditioning to auth.setup.ts file to use existing auth json file if it's applicable
+-Added some optimizations to playwright config file
+-Cleaned up gh action for playwright tests (removed Cypress references)
+-Updating gh workflow for PR automation with extra test sharding
+-General cleanup of redundant code
diff --git a/.github/actions/run-pw-tests/action.yml b/.github/actions/run-pw-tests/action.yml
@@ -22,9 +22,15 @@ inputs:
   MAILPITURL:
     description: "mailpit uri"
     required: true
-  URL_TO_RUN: 
+  URL_TO_RUN:
     description: "Url which will be passed to testmo where can be found artifacts of the run"
     required: false
+  PW_WORKERS:
+    description: "Playwright workers"
+    required: true
+  PW_RETRIES:
+    description: "Playwright retries"
+    required: true
 
 runs:
   using: "composite"
@@ -50,8 +56,10 @@ runs:
         E2E_USER_PASSWORD: ${{ inputs.E2E_USER_PASSWORD }}
         E2E_PERMISSIONS_USERS_PASSWORD: ${{ inputs.E2E_PERMISSIONS_USERS_PASSWORD }}
         SHARD_NUMBER: ${{ inputs.SHARD }}
-        MAILPITURL: ${{ inputs.MAILPITURL }} 
-        URL_TO_RUN: ${{ inputs.URL_TO_RUN }} 
+        MAILPITURL: ${{ inputs.MAILPITURL }}
+        URL_TO_RUN: ${{ inputs.URL_TO_RUN }}
+        WORKERS: ${{ inputs.PW_WORKERS }}
+        RETRIES: ${{ inputs.PW_RETRIES }}
       run: npm run qa:pw-e2e -- --shard "$SHARD_NUMBER"
 
     - name: Upload blob report to GitHub Actions Artifacts

diff --git a/.github/workflows/pr-automation.yml b/.github/workflows/pr-automation.yml
@@ -118,13 +118,14 @@ jobs:
           deployment_id: ${{ steps.deployment.outputs.deployment_id }}
           env: ${{ needs.initialize-cloud.outputs.POOL_NAME }}
 
+
   run-tests:
     runs-on: ubuntu-22.04
     needs: [initialize-cloud, deploy-dashboard]
     strategy:
       fail-fast: false
       matrix:
-        shard: [1/2, 2/2]
+        shard: [1/4, 2/4, 3/4, 4/4]
     steps:
       - uses: actions/checkout@v4
 
@@ -134,10 +135,12 @@ jobs:
           SHARD: ${{ matrix.shard }}
           BASE_URL: ${{ needs.initialize-cloud.outputs.BASE_URL }}
           API_URL: ${{ needs.initialize-cloud.outputs.API_URL }}
-          E2E_USER_NAME: ${{ secrets.CYPRESS_USER_NAME }}
-          E2E_USER_PASSWORD: ${{ secrets.CYPRESS_USER_PASSWORD }}
-          E2E_PERMISSIONS_USERS_PASSWORD: ${{ secrets.CYPRESS_PERMISSIONS_USERS_PASSWORD }}
-          MAILPITURL: ${{ secrets.CYPRESS_MAILPITURL }}
+          E2E_USER_NAME: ${{ secrets.E2E_USER_NAME }}
+          E2E_USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
+          E2E_PERMISSIONS_USERS_PASSWORD: ${{ secrets.E2E_PERMISSIONS_USERS_PASSWORD }}
+          MAILPITURL: ${{ secrets.MAILPITURL }}
+          PW_WORKERS: ${{ vars.PW_WORKERS }}
+          PW_RETRIES: ${{ vars.PW_RETRIES }}
 
   merge-reports:
     if: "!cancelled()"
@@ -147,4 +150,4 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Merge playwright reports
-        uses: ./.github/actions/merge-pw-reports
+        uses: ./.github/actions/merge-pw-reports
diff --git a/.github/workflows/run-test-cron.yml b/.github/workflows/run-test-cron.yml
@@ -74,10 +74,12 @@ jobs:
           SHARD: ${{ matrix.shard }}
           BASE_URL: ${{ needs.initialize-cloud.outputs.BASE_URL }}
           API_URL: ${{ needs.initialize-cloud.outputs.API_URL }}
-          E2E_USER_NAME: ${{ secrets.CYPRESS_USER_NAME }}
-          E2E_USER_PASSWORD: ${{ secrets.CYPRESS_USER_PASSWORD }}
-          E2E_PERMISSIONS_USERS_PASSWORD: ${{ secrets.CYPRESS_PERMISSIONS_USERS_PASSWORD }}
-          MAILPITURL: ${{ secrets.CYPRESS_MAILPITURL }}
+          E2E_USER_NAME: ${{ secrets.E2E_USER_NAME }}
+          E2E_USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
+          E2E_PERMISSIONS_USERS_PASSWORD: ${{ secrets.E2E_PERMISSIONS_USERS_PASSWORD }}
+          MAILPITURL: ${{ secrets.MAILPITURL }}
+          PW_WORKERS: ${{ vars.PW_WORKERS }}
+          PW_RETRIES: ${{ vars.PW_RETRIES }}
           URL_TO_RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
       - name: submit-results-to-testmo
@@ -97,7 +99,7 @@ jobs:
 
       - name: Merge playwright reports
         uses: ./.github/actions/merge-pw-reports
-      
+
       - name: complete testmo report
         uses: ./.github/actions/testmo/testmo-finish
         with:
@@ -122,6 +124,6 @@ jobs:
           --slack_webhook_url "$slack_webhook_url" \
           --environment "$environment" \
           --url_to_action "$url_to_action" \
-          --ref_name "$ref_name" 
-        
+          --ref_name "$ref_name"
+
 
diff --git a/.github/workflows/run-test-manual.yml b/.github/workflows/run-test-manual.yml
@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           sparse-checkout: ./.github/actions
-    
+
       - name: Set variables mode
         id: set_variables_mode
         shell: bash
@@ -64,7 +64,7 @@ jobs:
           testmoToken: ${{ secrets.TESTMO_TOKEN }}
           testmoRunName: "Playwright run ${{github.ref_name}}"
         id: init-testmo
-    
+
   run-tests:
     runs-on: ubuntu-22.04
     needs: ["initialize-cloud", "create-run-on-testmo"]
@@ -81,10 +81,12 @@ jobs:
           SHARD: ${{ matrix.shard }}
           BASE_URL: ${{ needs.initialize-cloud.outputs.BASE_URL }}
           API_URL: ${{ needs.initialize-cloud.outputs.API_URL }}
-          E2E_USER_NAME: ${{ secrets.CYPRESS_USER_NAME }}
-          E2E_USER_PASSWORD: ${{ secrets.CYPRESS_USER_PASSWORD }}
-          E2E_PERMISSIONS_USERS_PASSWORD: ${{ secrets.CYPRESS_PERMISSIONS_USERS_PASSWORD }}
-          MAILPITURL: ${{ secrets.CYPRESS_MAILPITURL }}
+          E2E_USER_NAME: ${{ secrets.E2E_USER_NAME }}
+          E2E_USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
+          E2E_PERMISSIONS_USERS_PASSWORD: ${{ secrets.E2E_PERMISSIONS_USERS_PASSWORD }}
+          MAILPITURL: ${{ secrets.MAILPITURL }}
+          PW_WORKERS: ${{ vars.PW_WORKERS }}
+          PW_RETRIES: ${{ vars.PW_RETRIES }}
           URL_TO_RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
       - name: submit-results-to-testmo
@@ -129,4 +131,4 @@ jobs:
           --slack_webhook_url "$slack_webhook_url" \
           --environment "$environment" \
           --url_to_action "$url_to_action" \
-          --ref_name "$ref_name" 
+          --ref_name "$ref_name"
diff --git a/playwright.config.ts b/playwright.config.ts
@@ -1,17 +1,17 @@
 import dotenv from "dotenv";
-
 import { defineConfig, devices } from "@playwright/test";
 
 dotenv.config();
-
+const env = process.env;
+const DEFAULT_RETRIES = '1';
+const DEFAULT_WORKERS = '2';
 export default defineConfig({
   testDir: "playwright/tests",
   fullyParallel: true,
-  forbidOnly: !!process.env.CI,
-  // TODO hardcoded values should be extracted to ENVs
-  retries: process.env.CI ? 2 : 0,
-  workers: process.env.CI ? 2 : 2,
-  reporter: process.env.CI
+  forbidOnly: !!env.CI,
+  retries: parseInt(env.RETRIES || DEFAULT_RETRIES),
+  workers: parseInt(env.WORKERS || DEFAULT_WORKERS),
+    reporter: process.env.CI
     ? [
         ["blob"],
         ["github"],
@@ -29,30 +29,25 @@ export default defineConfig({
         ],
       ]
     : [["html"], ["list"]],
-  timeout: 60000,
   expect: { timeout: 10000 },
-  // webServer: {
-  //   command: "npm run dev",
-  //   url: "http://localhost:9000/",
-  //   reuseExistingServer: !process.env.CI,
-  // },
-  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  maxFailures: 5,
+  timeout: env.CI ? 45000 : 60000,
   use: {
-    baseURL: process.env.BASE_URL,
-    trace: "on-first-retry",
+    baseURL: env.BASE_URL || '',
+    trace: env.CI ? "on-all-retries" : "on",
     screenshot: "only-on-failure",
     testIdAttribute: "data-test-id",
-    video: process.env.CI ? "retain-on-failure" : "off",
+    video: env.CI ? "retain-on-failure" : "off",
+    headless: true,
   },
-
-  /* Configure projects for major browsers */
   projects: [
-    { name: "setup", testMatch: /.*\.setup\.ts/ },
-
     {
-      // if new project added make sure to add dependency as below
-      dependencies: ["setup"],
+      name: "setup",
+      testMatch: /.*\.setup\.ts/
+    },
+    {
       name: "chromium",
+      dependencies: ["setup"],
       use: { ...devices["Desktop Chrome"] },
     },
   ],

diff --git a/playwright/pages/basePage.ts b/playwright/pages/basePage.ts
@@ -114,6 +114,11 @@ export class BasePage {
       "No error banner should be visible",
     ).not.toBeVisible();
   }
+  async waitForNetworkIdle(action: () => Promise<void>) {
+    const responsePromise = this.page.waitForResponse('**/graphql/');
+    await action();
+    await responsePromise;
+  }
   async resizeWindow(w: number, h: number) {
     await this.page.setViewportSize({width: w, height: h,});
   }
@@ -254,7 +259,7 @@ export class BasePage {
       .locator("tr")
       .filter({ hasText: expectedText })
       .first()
-      .waitFor({ state: "attached" });
+      .waitFor({ state: "attached", timeout: 10000 });
     const gridRowsWithText = await this.gridCanvas
       .locator("tr")
       .filter({ hasText: expectedText })

diff --git a/playwright/pages/dialogs/productCreateDialog.ts b/playwright/pages/dialogs/productCreateDialog.ts
@@ -1,8 +1,7 @@
+import { BasePage } from "@pages/basePage";
 import type { Page } from "@playwright/test";
-import {expect} from "@playwright/test";
 
-export class ProductCreateDialog {
-  readonly page: Page;
+export class ProductCreateDialog extends BasePage{
 
   constructor(
     page: Page,
@@ -16,12 +15,10 @@ export class ProductCreateDialog {
     readonly confirmButton = page.getByTestId("submit"),
     readonly tooltipResult = page.getByRole("tooltip"),
   ) {
-    this.page = page;
+    super(page);
   }
   async selectProductTypeWithVariants(productType: string = "Beer") {
-    const responsePromise = this.page.waitForResponse('**/graphql/');
-    await this.dialogProductTypeInput.fill(productType);
-    await responsePromise;
+    await this.waitForNetworkIdle(() => this.dialogProductTypeInput.fill(productType));
     await this.promptedOptions.filter({hasText:productType}).click();
   }
   async clickConfirmButton() {

diff --git a/playwright/tests/auth.setup.ts b/playwright/tests/auth.setup.ts
@@ -22,20 +22,26 @@ const authenticateAndSaveState = async (request: APIRequestContext, email: strin
     });
     fs.writeFileSync(filePath, JSON.stringify(loginJsonInfo, null, 2));
 };
-const authSetup = async ( request: APIRequestContext,email: string, password: string, fileName: string) => {
+
+const authSetup = async (request: APIRequestContext, email: string, password: string, fileName: string) => {
     const tempDir = path.join(__dirname, '../.auth');
     if (!fs.existsSync(tempDir)) {
         fs.mkdirSync(tempDir, { recursive: true });
     }
     const tempFilePath = path.join(tempDir, fileName);
-    await authenticateAndSaveState(request, email, password, tempFilePath);
+
+    if (!fs.existsSync(tempFilePath)) {
+        await authenticateAndSaveState(request, email, password, tempFilePath);
+    }
 };
+
 setup("Authenticate as admin via API", async ({ request }) => {
     await authSetup(request, process.env.E2E_USER_NAME!, process.env.E2E_USER_PASSWORD!, 'admin.json');
 });
 
 const user: UserPermissionType = USER_PERMISSION;
 const password: string = process.env.E2E_PERMISSIONS_USERS_PASSWORD!;
+
 for (const permission of permissions) {
     setup(`Authenticate as ${permission} user via API`, async ({ request }) => {
         await authSetup(request, user[permission], password, `${permission}.json`);

diff --git a/playwright/tests/giftCards.spec.ts b/playwright/tests/giftCards.spec.ts
@@ -37,7 +37,7 @@ test("TC: SALEOR_105 Issue gift card @e2e @gift", async () => {
     state: "hidden",
     timeout: 30000,
   });
-  await giftCardsPage.gotoGiftCardsListView();
+  await giftCardsPage.waitForNetworkIdle(() =>  giftCardsPage.gotoGiftCardsListView());
   await giftCardsPage.waitForDOMToFullyLoad();
   const actualNumberOfRows = await giftCardsPage.getNumberOfGridRowsWithText(
     "Code",
@@ -65,10 +65,7 @@ test("TC: SALEOR_106 Issue gift card with specific customer and expiry date @e2e
     state: "hidden",
     timeout: 30000,
   });
-
-  await giftCardsPage.waitForGrid();
-
-  await giftCardsPage.gotoGiftCardsListView();
+  await giftCardsPage.waitForNetworkIdle(() =>  giftCardsPage.gotoGiftCardsListView());
   await giftCardsPage.waitForDOMToFullyLoad();
   const actualNumberOfRows = await giftCardsPage.getNumberOfGridRowsWithText(
     "Code",
@@ -129,7 +126,7 @@ test("TC: SALEOR_111 Bulk delete gift cards @e2e @gift", async () => {
   await giftCardsPage.deleteDialog.clickConfirmDeletionCheckbox();
   await giftCardsPage.deleteDialog.clickDeleteButton();
   await giftCardsPage.dialog.waitFor({ state: "hidden" });
-  await giftCardsPage.gotoGiftCardsListView();
+  await giftCardsPage.waitForNetworkIdle(() =>  giftCardsPage.gotoGiftCardsListView());
   await giftCardsPage.waitForDOMToFullyLoad();
   const actualNumberOfRows = await giftCardsPage.getNumberOfGridRowsWithText(
     "Code",