[test optimization] Improve playwright flakiness (this time for real?) (#5440)

Author: juan-fernandez

.github/workflows/project.yml

@@ -61,11 +61,32 @@ jobs:
         env:
           DD_INJECTION_ENABLED: 'true'
 
+  integration-playwright:
+    strategy:
+      matrix:
+        version: [18, latest]
+    runs-on: ubuntu-latest
+    env:
+      DD_SERVICE: dd-trace-js-integration-tests
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: 1
+      DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
+        with:
+          node-version: ${{ matrix.version }}
+      - uses: ./.github/actions/install
+      # Install system dependencies for playwright
+      - run: npx playwright install-deps
+      - run: yarn test:integration:playwright
+        env:
+          NODE_OPTIONS: '-r ./ci/init'
+
   integration-ci:
     strategy:
       matrix:
         version: [18, latest]
-        framework: [cucumber, playwright, selenium, jest, mocha]
+        framework: [cucumber, selenium, jest, mocha]
     runs-on: ubuntu-latest
     env:
       DD_SERVICE: dd-trace-js-integration-tests

integration-tests/ci-visibility/playwright-tests-test-capabilities/passing-test.js

@@ -0,0 +1,11 @@
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test('should work with passing tests', async ({ page }) => {
+  await expect(page.locator('.hello-world')).toHaveText([
+    'Hello World'
+  ])
+})

integration-tests/playwright/playwright.spec.js

@@ -47,15 +47,14 @@ versions.forEach((version) => {
     let sandbox, cwd, receiver, childProcess, webAppPort
 
     before(async function () {
-      // bump from 60 to 90 seconds because we also need to install dependencies and download chromium
+      // bump from 60 to 90 seconds because playwright is heavy
       this.timeout(90000)
       sandbox = await createSandbox([`@playwright/test@${version}`, 'typescript'], true)
       cwd = sandbox.folder
       const { NODE_OPTIONS, ...restOfEnv } = process.env
-      // Install system dependencies
-      execSync('npx playwright install-deps', { cwd, env: restOfEnv })
       // Install chromium (configured in integration-tests/playwright.config.js)
-      execSync('npx playwright install', { cwd, env: restOfEnv })
+      // *Be advised*: this means that we'll only be using chromium for this test suite
+      execSync('npx playwright install chromium', { cwd, env: restOfEnv, stdio: 'inherit' })
       webAppPort = await getPort()
       webAppServer.listen(webAppPort)
     })
@@ -202,7 +201,9 @@ versions.forEach((version) => {
       })
     })
 
-    it('works when tests are compiled to a different location', (done) => {
+    it('works when tests are compiled to a different location', function (done) {
+      // this has shown some flakiness
+      this.retries(1)
       let testOutput = ''
 
       receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => {
@@ -215,7 +216,7 @@ versions.forEach((version) => {
         assert.include(testOutput, '1 passed')
         assert.include(testOutput, '1 skipped')
         assert.notInclude(testOutput, 'TypeError')
-      }).then(() => done()).catch(done)
+      }, 25000).then(() => done()).catch(done)
 
       childProcess = exec(
         'node ./node_modules/typescript/bin/tsc' +
@@ -1082,6 +1083,15 @@ versions.forEach((version) => {
 
     context('libraries capabilities', () => {
       it('adds capabilities to tests', (done) => {
+        receiver.setKnownTests(
+          {
+            playwright: {
+              'passing-test.js': [
+                'should work with passing tests'
+              ]
+            }
+          }
+        )
         receiver.setSettings({
           flaky_test_retries_enabled: true,
           itr_enabled: false,
@@ -1111,12 +1121,14 @@ versions.forEach((version) => {
             env: {
               ...getCiVisAgentlessConfig(receiver.port),
               PW_BASE_URL: `http://localhost:${webAppPort}`,
+              TEST_DIR: './ci-visibility/playwright-tests-test-capabilities',
               DD_TEST_SESSION_NAME: 'my-test-session-name'
             },
             stdio: 'pipe'
           }
         )
-        childProcess.on('exit', (exitCode) => {
+
+        childProcess.on('exit', () => {
           eventsPromise.then(() => {
             done()
           }).catch(done)