[test optimization] Clean up handling of known tests in playwright (#6413)

Author: juan-fernandez

integration-tests/playwright/playwright.spec.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { once } = require('node:events')
 const { exec, execSync } = require('child_process')
 const satisfies = require('semifies')
 const path = require('path')
@@ -25,6 +26,7 @@ const {
   TEST_IS_NEW,
   TEST_IS_RETRY,
   TEST_EARLY_FLAKE_ENABLED,
+  TEST_EARLY_FLAKE_ABORT_REASON,
   TEST_SUITE,
   TEST_CODE_OWNERS,
   TEST_SESSION_NAME,
@@ -537,7 +539,9 @@ versions.forEach((version) => {
         })
 
         receiver.setKnownTestsResponseCode(500)
-        receiver.setKnownTests({})
+        receiver.setKnownTests({
+          playwright: {}
+        })
 
         const receiverPromise = receiver
           .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
@@ -641,6 +645,61 @@ versions.forEach((version) => {
           receiverPromise.then(() => done()).catch(done)
         })
       })
+
+      it('does not run EFD if the known tests response is invalid', async () => {
+        receiver.setSettings({
+          early_flake_detection: {
+            enabled: true,
+            slow_test_retries: {
+              '5s': NUM_RETRIES_EFD
+            }
+          },
+          known_tests_enabled: true
+        })
+
+        receiver.setKnownTests(
+          {
+            'not-playwright': {}
+          }
+        )
+
+        const receiverPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+
+            const testSession = events.find(event => event.type === 'test_session_end').content
+            assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED)
+            assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')
+
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            const newTests = tests.filter(test =>
+              test.resource.endsWith('should work with passing tests')
+            )
+            newTests.forEach(test => {
+              assert.notProperty(test.meta, TEST_IS_NEW)
+            })
+
+            const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
+            assert.equal(retriedTests.length, 0)
+          })
+
+        childProcess = exec(
+          './node_modules/.bin/playwright test -c playwright.config.js',
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              PW_BASE_URL: `http://localhost:${webAppPort}`
+            },
+            stdio: 'pipe'
+          }
+        )
+
+        await Promise.all([
+          once(childProcess, 'exit'),
+          receiverPromise,
+        ])
+      })
     })
 
     it('does not crash when maxFailures=1 and there is an error', (done) => {
@@ -1703,7 +1762,9 @@ versions.forEach((version) => {
 
       context('test is new', () => {
         it('should be retried and marked both as new and modified', (done) => {
-          receiver.setKnownTests({})
+          receiver.setKnownTests({
+            playwright: {}
+          })
           receiver.setSettings({
             impacted_tests_enabled: true,
             early_flake_detection: {

packages/datadog-instrumentations/src/playwright.js

@@ -51,6 +51,7 @@ let remainingTestsByFile = {}
 let isKnownTestsEnabled = false
 let isEarlyFlakeDetectionEnabled = false
 let earlyFlakeDetectionNumRetries = 0
+let isEarlyFlakeDetectionFaulty = false
 let isFlakyTestRetriesEnabled = false
 let flakyTestRetriesCount = 0
 let knownTests = {}
@@ -64,6 +65,10 @@ let quarantinedButNotAttemptToFixFqns = new Set()
 let rootDir = ''
 const MINIMUM_SUPPORTED_VERSION_RANGE_EFD = '>=1.38.0' // TODO: remove this once we drop support for v5
 
+function isValidKnownTests (receivedKnownTests) {
+  return !!receivedKnownTests.playwright
+}
+
 function getTestFullyQualifiedName (test) {
   const fullname = getTestFullname(test)
   return `${test._requireFile} ${fullname}`
@@ -79,8 +84,11 @@ function getTestProperties (test) {
 }
 
 function isNewTest (test) {
+  if (!isValidKnownTests(knownTests)) {
+    return false
+  }
   const testSuite = getTestSuitePath(test._requireFile, rootDir)
-  const testsForSuite = knownTests?.playwright?.[testSuite] || []
+  const testsForSuite = knownTests.playwright[testSuite] || []
 
   return !testsForSuite.includes(getTestFullname(test))
 }
@@ -557,6 +565,11 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
         } else {
           knownTests = receivedKnownTests
         }
+        if (!isValidKnownTests(receivedKnownTests)) {
+          isEarlyFlakeDetectionFaulty = true
+          isEarlyFlakeDetectionEnabled = false
+          isKnownTestsEnabled = false
+        }
       } catch (err) {
         isEarlyFlakeDetectionEnabled = false
         isKnownTestsEnabled = false
@@ -652,6 +665,7 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
     testSessionFinishCh.publish({
       status: STATUS_TO_TEST_STATUS[sessionStatus],
       isEarlyFlakeDetectionEnabled,
+      isEarlyFlakeDetectionFaulty,
       isTestManagementTestsEnabled,
       onDone
     })

packages/datadog-plugin-playwright/src/index.js

@@ -17,6 +17,7 @@ const {
   TEST_IS_NEW,
   TEST_IS_RETRY,
   TEST_EARLY_FLAKE_ENABLED,
+  TEST_EARLY_FLAKE_ABORT_REASON,
   TELEMETRY_TEST_SESSION,
   TEST_RETRY_REASON,
   TEST_MANAGEMENT_IS_QUARANTINED,
@@ -70,6 +71,7 @@ class PlaywrightPlugin extends CiPlugin {
     this.addSub('ci:playwright:session:finish', ({
       status,
       isEarlyFlakeDetectionEnabled,
+      isEarlyFlakeDetectionFaulty,
       isTestManagementTestsEnabled,
       onDone
     }) => {
@@ -79,7 +81,9 @@ class PlaywrightPlugin extends CiPlugin {
       if (isEarlyFlakeDetectionEnabled) {
         this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true')
       }
-
+      if (isEarlyFlakeDetectionFaulty) {
+        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')
+      }
       if (this.numFailedSuites > 0) {
         let errorMessage = `Test suites failed: ${this.numFailedSuites}.`
         if (this.numFailedTests > 0) {