refactor(test): use @playwright/test instead of jasmine (#4260)

.github/workflows/ci.yml

@@ -14,7 +14,6 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: "14.x"
-      - uses: microsoft/playwright-github-action@v1
       - id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
       - uses: actions/cache@v1
@@ -24,6 +23,8 @@ jobs:
           restore-keys: |
             yarn-
       - run: yarn --frozen-lockfile
+      - run: npx playwright install-deps
+      - run: npx playwright install
       - run: yarn ci && yarn scripts:test
       - uses: codecov/codecov-action@v1
         with:

.gitignore

@@ -18,6 +18,7 @@ npm-debug.log
 coverage
 coverage-e2e/
 coverage-e2e-*/
+.nyc_output
 demo/src/api-docs.ts
 demo/src/public/**/plnkr.html
 demo/src/public/**/stackblitz.html

demo/playwright.config.ts

@@ -0,0 +1,24 @@
+import {PlaywrightTestConfig} from '@playwright/test';
+
+export const baseURL = "http://localhost:4200/#";
+
+const config: PlaywrightTestConfig = {
+  reporter: process.env.CI ? 'github' : 'list',
+  testDir: "src",
+  testMatch: /\.e2e-spec\.ts$/,
+  timeout: 60000,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  webServer: {
+    command: "yarn demo -c playwright",
+    timeout: 300000,
+    url: baseURL,
+    reuseExistingServer: !process.env.CI,
+  },
+  use: {
+    baseURL,
+    viewport: {width: 1280, height: 720},
+  }
+};
+
+export default config;

demo/src/app/components/components.e2e-spec.ts

@@ -1,5 +1,4 @@
-import {BASE_URL, test} from '../../../playwright.conf';
-import {ConsoleMessage, Page} from 'playwright';
+import {ConsoleMessage, test, expect} from '@playwright/test';
 
 const SELECTOR_TAB_LINKS = 'header.title a.nav-link';
 const SELECTOR_SIDE_NAV_COMPONENT_LINKS = 'ngbd-side-nav >> a[href^="#/components/"]';
@@ -10,60 +9,57 @@ const COMPONENTS = [
   'popover', 'progressbar', 'rating', 'table', 'tabset', 'timepicker', 'toast', 'tooltip', 'typeahead'
 ];
 
-describe(`Components`, () => {
+test.describe(`Components`, () => {
 
   let messages: ConsoleMessage[] = [];
-  let page: Page;
 
-  beforeAll(async () => {
-    page = await test.newPage(`${BASE_URL}/components`);
+  test.beforeEach(async({page, baseURL}) => {
+    messages = [];
+    await page.goto(`${baseURL}/components`);
     page.on('console', msg => messages.push(msg));
 
     await page.waitForSelector(SELECTOR_SIDE_NAV_COMPONENT_LINKS);
   });
 
-  afterAll(async() => {
-    await test.destroy();
-  }, 120000);
-
-  it('should cover all components we have', async() => {
-    for (const link of await page.$$(SELECTOR_SIDE_NAV_COMPONENT_LINKS)) {
-      const componentName = await link.innerText();
-      expect(COMPONENTS).toContain(componentName.toLowerCase(), `'${componentName}' is not covered by demo e2e tests`);
+  test.afterEach(async() => {
+    if (messages.length > 0) {
+      console.log(messages);
+      test.fail(true, 'Unexpected console messages found');
     }
   });
 
-  COMPONENTS.forEach(component => {
-    describe(`${component} page`, () => {
+  test('should load successfully', async({page}) => {
+
+    await test.step('should cover all components we have', async() => {
+      for (const link of await page.$$(SELECTOR_SIDE_NAV_COMPONENT_LINKS)) {
+        const componentName = await link.innerText();
+        expect(COMPONENTS, `'${componentName}' is not covered by demo e2e tests`)
+            .toContain(componentName.toLowerCase());
+      }
+    });
 
-      const SELECTOR_EXAMPLE_LINK = `a[href="#/components/${component}/examples"]`;
-      const SELECTOR_SIDE_NAV_COMPONENT_LINK = `ngbd-side-nav >> a[href^="#/components/${component}"]`;
+    for (const component of COMPONENTS) {
+      await test.step(`${component} page`, async() => {
+
+        const SELECTOR_EXAMPLE_LINK = `a[href="#/components/${component}/examples"]`;
+        const SELECTOR_SIDE_NAV_COMPONENT_LINK = `ngbd-side-nav >> a[href^="#/components/${component}"]`;
 
-      beforeAll(async() => {
-        messages = [];
         await page.click(SELECTOR_SIDE_NAV_COMPONENT_LINK);
         await page.waitForSelector(SELECTOR_TAB_LINKS);
-      });
 
-      afterAll(async() => {
-        if (messages.length > 0) {
-          console.log(messages);
-          fail('Unexpected console messages found');
-        }
-      });
-
-      it(`should display the tabs`, async() => {
-        for (const link of await page.$$(SELECTOR_TAB_LINKS)) {
-          await link.click();
-        }
-      });
+        await test.step(`should display the tabs`, async() => {
+          for (const link of await page.$$(SELECTOR_TAB_LINKS)) {
+            await link.click();
+          }
+        });
 
-      it(`should display code samples`, async() => {
-        await page.click(SELECTOR_EXAMPLE_LINK);
-        for (const link of await page.$$(SELECTOR_CODE_BUTTONS)) {
-          await link.click();
-        }
+        await test.step(`should display code samples`, async() => {
+          await page.click(SELECTOR_EXAMPLE_LINK);
+          for (const link of await page.$$(SELECTOR_CODE_BUTTONS)) {
+            await link.click();
+          }
+        });
       });
-    });
+    }
   });
 });

e2e-app/baseTest.ts

@@ -0,0 +1,55 @@
+import {Page, test as baseTest} from "@playwright/test";
+import {randomBytes} from "crypto";
+import {promises as fs} from "fs";
+import {join} from "path";
+
+let globalPage: Page | null = null;
+export const getPage = (): Page => {
+  return globalPage;
+};
+export const setPage = (page: Page | null) => {
+  globalPage = page;
+};
+let globalBrowserName: 'chromium' | 'firefox' | 'webkit' | null = null;
+export const getBrowserName = () => {
+  return globalBrowserName;
+};
+
+export interface Fixtures {
+  testURL: string;
+  testSelector: string;
+}
+
+export const test = baseTest.extend<Fixtures>({
+  testURL: "",
+  testSelector: "",
+  page: async({page, baseURL, testURL, testSelector, browserName}, use) => {
+    globalBrowserName = browserName;
+    if (!testURL || !testSelector) {
+      throw new Error("testURL and testSelector must be defined!");
+    }
+
+    // Listen for all console events and handle errors
+    page.on('console', async msg => {
+      const type = msg.type();
+      if (type === 'error' || type === 'warning') {
+        test.fail(true, msg.text());
+      }
+    });
+
+    // Log all uncaught errors to the terminal
+    page.on('pageerror', exception => { console.log(`Uncaught exception:\n${exception}`); });
+
+    await page.goto(`${baseURL}/${testURL}`);
+    await page.waitForSelector(testSelector);
+
+    await use(page);
+
+    const coverage: string = await page.evaluate('JSON.stringify(window.__coverage__);');
+    if (coverage) {
+      const name = randomBytes(32).toString("hex");
+      await fs.writeFile(join(__dirname, '..', '.nyc_output', `${name}.json`), coverage);
+    }
+    globalPage = null;
+  },
+});

e2e-app/playwright.config.ts

@@ -0,0 +1,45 @@
+import {PlaywrightTestConfig, devices} from '@playwright/test';
+
+export const baseURL = "http://localhost:4200/#";
+
+const config: PlaywrightTestConfig = {
+  reporter: process.env.CI ? 'github' : 'list',
+  globalSetup: require.resolve('./setup.e2e-spec'),
+  testDir: "src",
+  testMatch: /\.e2e-spec\.ts$/,
+  timeout: 60000,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  webServer: {
+    command: "yarn e2e-app:serve -c playwright",
+    timeout: 300000,
+    url: baseURL,
+    reuseExistingServer: !process.env.CI,
+  },
+  use: {
+    baseURL,
+    viewport: {width: 1280, height: 720},
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+          ...devices['Desktop Chrome'],
+      },
+    },
+    {
+      name: 'firefox',
+      use: {
+          ...devices['Desktop Firefox'],
+      },
+    },
+    {
+      name: 'webkit',
+      use: {
+          ...devices['Desktop Safari'],
+      },
+    },
+  ]
+};
+
+export default config;

e2e-app/setup.e2e-spec.ts

@@ -1,57 +1,16 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-import * as fs from 'fs';
-import * as path from 'path';
-import {browserName, test, launchOptions} from './playwright.conf';
+import {FullConfig} from "@playwright/test";
+import {promises as fs} from "fs";
+import {join} from "path";
+import NYC from "nyc";
 
-beforeAll(async() => {
-  Error.stackTraceLimit = Infinity;
-  jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000;
-  try {
-    console.log(`Launch browser ${browserName} with`, launchOptions);
-    await test.newPage();
-  } catch (e) {
-    console.error('Unable to setup a new page with playwright', e);
-  }
+async function globalSetup(config: FullConfig) {
+  await fs.mkdir(join(__dirname, "..", ".nyc_output"));
+  return async() => {
+    const nycInstance =
+        new NYC({cwd: join(__dirname, '..'), reportDir: `coverage-e2e`, reporter: ['lcov', 'json', 'text-summary']});
+    nycInstance.report();
+    nycInstance.cleanup();
+  };
+}
 
-  // Need to wait to avoid connection failures between Playwright and the browser.
-  // To remove once Playwright test runner will be used.
-  await test.page.waitForTimeout(500);
-
-  // Listen for all console events and handle errors
-  test.page.on('console', async msg => {
-    const type = msg.type();
-    if (type === 'error' || type === 'warning') {
-      fail(msg.text());
-    }
-  });
-
-  // Log all uncaught errors to the terminal
-  test.page.on('pageerror', exception => { console.log(`Uncaught exception:\n${exception}`); });
-
-});
-
-afterAll(async() => {
-  if (browserName === 'chromium') {
-    try {
-      console.log('Retrieving coverage...');
-      const coverage: string = await test.page.evaluate('JSON.stringify(window.__coverage__);');
-      if (coverage) {
-        console.log(`Coverage retrieved (${coverage.length} bytes)`);
-        fs.mkdirSync(path.join(__dirname, '..', '.nyc_output'));
-        fs.writeFileSync(path.join(__dirname, '..', '.nyc_output', 'out.json'), coverage);
-        const NYC = require('nyc');
-        const nycInstance = new NYC(
-            {cwd: path.join(__dirname, '..'), reportDir: `coverage-e2e`, reporter: ['lcov', 'json', 'text-summary']});
-        nycInstance.report();
-        nycInstance.cleanup();
-        console.log('Coverage saved successfully!');
-      } else {
-        console.log('No coverage data!');
-      }
-    } catch (error) {
-      console.log('Error in onComplete:', error);
-      process.exit(1);
-    }
-  }
-  await test.destroy();
-}, 120000);
+export default globalSetup;