[test optimization] Playwright + RUM: wait for RUM data to flush (#6759)

Author: juan-fernandez

integration-tests/ci-visibility/playwright-tests-rum/rum-multiple-goto-test.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test.describe('playwright', () => {
+  test('should have RUM active', async ({ page }) => {
+    await expect(page.locator('.hello-world')).toHaveText([
+      'Hello World'
+    ])
+    await page.goto(`${process.env.PW_BASE_URL}/another-page`)
+    await expect(page.locator('.hello-world')).toHaveText([
+      'Hello World'
+    ])
+  })
+})

integration-tests/ci-visibility/playwright-tests-rum/active-test-span-rum-test.js renamed to integration-tests/ci-visibility/playwright-tests-rum/rum-test.js

@@ -1700,25 +1700,26 @@ versions.forEach((version) => {
         receiver
           .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
             const events = payloads.flatMap(({ payload }) => payload.events)
-            const playwrightTest = events.find(event => event.type === 'test').content
-            if (isRedirecting) {
-              assert.notProperty(playwrightTest.meta, TEST_IS_RUM_ACTIVE)
-              assert.notProperty(playwrightTest.meta, TEST_BROWSER_VERSION)
-            } else {
-              assert.property(playwrightTest.meta, TEST_IS_RUM_ACTIVE, 'true')
-              assert.property(playwrightTest.meta, TEST_BROWSER_VERSION)
-            }
-            assert.include(playwrightTest.meta, {
-              [TEST_BROWSER_NAME]: 'chromium',
-              [TEST_TYPE]: 'browser'
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            tests.forEach(test => {
+              if (isRedirecting) {
+                // can't do assertions because playwright has been redirected
+                assert.propertyVal(test.meta, TEST_STATUS, 'fail')
+                assert.notProperty(test.meta, TEST_IS_RUM_ACTIVE)
+                assert.notProperty(test.meta, TEST_BROWSER_VERSION)
+              } else {
+                assert.propertyVal(test.meta, TEST_STATUS, 'pass')
+                assert.property(test.meta, TEST_IS_RUM_ACTIVE, 'true')
+                assert.property(test.meta, TEST_BROWSER_VERSION)
+              }
             })
           })
 
-      const runTest = (done, { isRedirecting }, extraEnvVars) => {
+      const runRumTest = async ({ isRedirecting }, extraEnvVars) => {
         const testAssertionsPromise = getTestAssertions({ isRedirecting })
 
         childProcess = exec(
-          './node_modules/.bin/playwright test -c playwright.config.js active-test-span-rum-test.js',
+          './node_modules/.bin/playwright test -c playwright.config.js',
           {
             cwd,
             env: {
@@ -1731,17 +1732,18 @@ versions.forEach((version) => {
           }
         )
 
-        childProcess.on('exit', () => {
-          testAssertionsPromise.then(() => done()).catch(done)
-        })
+        await Promise.all([
+          once(childProcess, 'exit'),
+          testAssertionsPromise
+        ])
       }
 
-      it('can correlate tests and RUM sessions', (done) => {
-        runTest(done, { isRedirecting: false })
+      it('can correlate tests and RUM sessions', async () => {
+        await runRumTest({ isRedirecting: false })
       })
 
-      it('do not crash when redirecting and RUM sessions are not active', (done) => {
-        runTest(done, { isRedirecting: true })
+      it('do not crash when redirecting and RUM sessions are not active', async () => {
+        await runRumTest({ isRedirecting: true })
       })
     })
 

integration-tests/playwright/playwright.spec.js

@@ -38,6 +38,8 @@ const testSuiteToTestStatuses = new Map()
 const testSuiteToErrors = new Map()
 const testsToTestStatuses = new Map()
 
+const RUM_FLUSH_WAIT_TIME = 1000
+
 let applyRepeatEachIndex = null
 
 let startedSuites = []
@@ -1134,16 +1136,19 @@ addHook({
                 })
 
                 if (isRumActive) {
+                  // Give some time RUM to flush data, similar to what we do in selenium
+                  await new Promise(resolve => setTimeout(resolve, RUM_FLUSH_WAIT_TIME))
                   const url = page.url()
                   if (url) {
                     const domain = new URL(url).hostname
                     await page.context().addCookies([{
                       name: 'datadog-ci-visibility-test-execution-id',
                       value: '',
                       domain,
-                      expires: 0,
                       path: '/'
                     }])
+                  } else {
+                    log.error('RUM is active but page.url() is not available')
                   }
                 }
               }

packages/datadog-instrumentations/src/playwright.js

@@ -47,6 +47,7 @@ const {
   TELEMETRY_EVENT_FINISHED
 } = require('../../dd-trace/src/ci-visibility/telemetry')
 const { appClosing: appClosingTelemetry } = require('../../dd-trace/src/telemetry')
+const log = require('../../dd-trace/src/log')
 
 class PlaywrightPlugin extends CiPlugin {
   static id = 'playwright'
@@ -174,7 +175,10 @@ class PlaywrightPlugin extends CiPlugin {
     }) => {
       const store = storage('legacy').getStore()
       const span = store && store.span
-      if (!span) return
+      if (!span) {
+        log.error('ci:playwright:test:page-goto: test span not found')
+        return
+      }
 
       if (isRumActive) {
         span.setTag(TEST_IS_RUM_ACTIVE, 'true')