[test optimization] Playwright: report tests that did not run (#6797)

Author: juan-fernandez

integration-tests/ci-visibility/playwright-did-not-run/did-not-run.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { test, expect } = require('@playwright/test')
+
+test.describe('did not run', () => {
+  test('because of early bail', async () => {
+    expect(true).toBe(false)
+  })
+})

integration-tests/ci-visibility/playwright-did-not-run/fail-test.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { test, expect } = require('@playwright/test')
+
+test.describe('failing test', () => {
+  test('fails and causes early bail', async () => {
+    expect(true).toBe(false)
+  })
+})

integration-tests/playwright.config.js

@@ -3,6 +3,26 @@
 // Playwright config file for integration tests
 const { devices } = require('@playwright/test')
 
+const projects = [
+  {
+    name: 'chromium',
+    use: {
+      ...devices['Desktop Chrome']
+    }
+  }
+]
+
+if (process.env.ADD_EXTRA_PLAYWRIGHT_PROJECT) {
+  projects.push({
+    name: 'extra-project',
+    use: {
+      ...devices['Desktop Chrome'],
+    },
+    dependencies: ['chromium'],
+    testMatch: 'did-not-run.js'
+  })
+}
+
 const config = {
   baseURL: process.env.PW_BASE_URL,
   testDir: process.env.TEST_DIR || './ci-visibility/playwright-tests',
@@ -11,14 +31,7 @@ const config = {
   workers: process.env.PLAYWRIGHT_WORKERS ? Number(process.env.PLAYWRIGHT_WORKERS) : undefined,
   reporter: 'line',
   /* Configure projects for major browsers */
-  projects: [
-    {
-      name: 'chromium',
-      use: {
-        ...devices['Desktop Chrome']
-      }
-    }
-  ],
+  projects,
   testMatch: '**/*-test.js'
 }
 

integration-tests/playwright/playwright.spec.js

@@ -2070,5 +2070,39 @@ versions.forEach((version) => {
         })
       })
     })
+
+    contextNewVersions('playwright early bail', () => {
+      it('reports tests that did not run', async () => {
+        const receiverPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            assert.equal(tests.length, 2)
+            const failedTest = tests.find(test => test.meta[TEST_STATUS] === 'fail')
+            assert.propertyVal(failedTest.meta, TEST_NAME, 'failing test fails and causes early bail')
+            const didNotRunTest = tests.find(test => test.meta[TEST_STATUS] === 'skip')
+            assert.propertyVal(didNotRunTest.meta, TEST_NAME, 'did not run because of early bail')
+          })
+
+        childProcess = exec(
+          './node_modules/.bin/playwright test -c playwright.config.js',
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              PW_BASE_URL: `http://localhost:${webAppPort}`,
+              TEST_DIR: './ci-visibility/playwright-did-not-run',
+              ADD_EXTRA_PLAYWRIGHT_PROJECT: 'true'
+            },
+            stdio: 'pipe'
+          }
+        )
+
+        await Promise.all([
+          once(childProcess, 'exit'),
+          receiverPromise
+        ])
+      })
+    })
   })
 })

packages/datadog-instrumentations/src/playwright.js

@@ -68,6 +68,8 @@ let modifiedFiles = {}
 const quarantinedOrDisabledTestsAttemptToFix = []
 let quarantinedButNotAttemptToFixFqns = new Set()
 let rootDir = ''
+let sessionProjects = []
+
 const MINIMUM_SUPPORTED_VERSION_RANGE_EFD = '>=1.38.0' // TODO: remove this once we drop support for v5
 
 function isValidKnownTests (receivedKnownTests) {
@@ -495,6 +497,7 @@ function dispatcherHook (dispatcherExport) {
     const dispatcher = this
     const worker = createWorker.apply(this, arguments)
     const projects = getProjectsFromDispatcher(dispatcher)
+    sessionProjects = projects
 
     // for older versions of playwright, `shouldCreateTestSpan` should always be true,
     // since the `_runTest` function wrapper is not available for older versions
@@ -535,6 +538,7 @@ function dispatcherHookNew (dispatcherExport, runWrapper) {
     const dispatcher = this
     const worker = createWorker.apply(this, arguments)
     const projects = getProjectsFromDispatcher(dispatcher)
+    sessionProjects = projects
 
     worker.on('testBegin', ({ testId }) => {
       const test = getTestByTestId(dispatcher, testId)
@@ -1255,3 +1259,46 @@ addHook({
 
   return workerPackage
 })
+
+function generateSummaryWrapper (generateSummary) {
+  return function () {
+    for (const test of this.suite.allTests()) {
+      // https://github.com/microsoft/playwright/blob/bf92ffecff6f30a292b53430dbaee0207e0c61ad/packages/playwright/src/reporters/base.ts#L279
+      const didNotRun = test.outcome() === 'skipped' &&
+        (!test.results.length || test.expectedStatus !== 'skipped')
+      if (didNotRun) {
+        const {
+          _requireFile: testSuiteAbsolutePath,
+          location: { line: testSourceLine },
+        } = test
+        const browserName = getBrowserNameFromProjects(sessionProjects, test)
+
+        testSkipCh.publish({
+          testName: getTestFullname(test),
+          testSuiteAbsolutePath,
+          testSourceLine,
+          browserName,
+        })
+      }
+    }
+    return generateSummary.apply(this, arguments)
+  }
+}
+
+// If a playwright project B has a dependency on project A,
+// and project A fails, the tests in project B will not run.
+// This hook is used to report tests that did not run as skipped.
+// Note: this is different from tests skipped via test.skip() or test.fixme()
+addHook({
+  name: 'playwright',
+  file: 'lib/reporters/base.js',
+  versions: ['>=1.38.0']
+}, (reportersPackage) => {
+  // v1.50.0 changed the name of the base reporter from BaseReporter to TerminalReporter
+  if (reportersPackage.TerminalReporter) {
+    shimmer.wrap(reportersPackage.TerminalReporter.prototype, 'generateSummary', generateSummaryWrapper)
+  } else if (reportersPackage.BaseReporter) {
+    shimmer.wrap(reportersPackage.BaseReporter.prototype, 'generateSummary', generateSummaryWrapper)
+  }
+  return reportersPackage
+})