[test optimization] [SDTEST-1630] Attempt to fix flaky tests implementation (#5429)

Author: Mariovido

integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature

@@ -0,0 +1,4 @@
+Feature: Attempt to fix
+  Scenario: Say attempt to fix
+    When the greeter says attempt to fix
+    Then I should have heard "attempt to fix"

integration-tests/ci-visibility/features-test-management/support/steps.js

@@ -1,6 +1,8 @@
 const assert = require('assert')
 const { When, Then } = require('@cucumber/cucumber')
 
+let numAttempt = 0
+
 Then('I should have heard {string}', function (expectedResponse) {
   if (this.whatIHeard === 'quarantine') {
     assert.equal(this.whatIHeard, 'fail')
@@ -21,3 +23,16 @@ When('the greeter says disabled', function () {
   // expected to fail if not disabled
   this.whatIHeard = 'disabld'
 })
+
+When('the greeter says attempt to fix', function () {
+  // eslint-disable-next-line no-console
+  console.log('I am running') // just to assert whether this is running
+  // expected to fail
+  if (process.env.SHOULD_ALWAYS_PASS) {
+    this.whatIHeard = 'attempt to fix'
+  } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+    this.whatIHeard = numAttempt++ % 2 === 0 ? 'attempt to fix' : 'attempt to fx'
+  } else {
+    this.whatIHeard = 'attempt to fx'
+  }
+})

integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js

@@ -0,0 +1,27 @@
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test.describe('attempt to fix', () => {
+  test('should attempt to fix failed test', async ({ page }) => {
+    let textToAssert
+
+    if (process.env.SHOULD_ALWAYS_PASS) {
+      textToAssert = 'Hello World'
+    } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+      // can't use numAttempt++ because we're running in parallel
+      if (Number(process.env.TEST_WORKER_INDEX) % 2 === 0) {
+        throw new Error('Hello Warld')
+      }
+      textToAssert = 'Hello World'
+    } else {
+      textToAssert = 'Hello Warld'
+    }
+
+    await expect(page.locator('.hello-world')).toHaveText([
+      textToAssert
+    ])
+  })
+})

integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js

@@ -0,0 +1,21 @@
+const { expect } = require('chai')
+
+let numAttempts = 0
+
+describe('attempt to fix tests', () => {
+  it('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running when attempt to fix') // to check if this is being run
+    if (process.env.SHOULD_ALWAYS_PASS) {
+      expect(1 + 2).to.equal(3)
+    } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+      if (numAttempts++ % 2 === 0) {
+        expect(1 + 2).to.equal(3)
+      } else {
+        expect(1 + 2).to.equal(4)
+      }
+    } else {
+      expect(1 + 2).to.equal(4)
+    }
+  })
+})

integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js

@@ -0,0 +1,9 @@
+const { expect } = require('chai')
+
+describe('attempt to fix tests 2', () => {
+  it('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running when attempt to fix 2') // to check if this is being run
+    expect(1 + 2).to.equal(3)
+  })
+})

integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs

@@ -0,0 +1,22 @@
+import { describe, test, expect } from 'vitest'
+
+let numAttempt = 0
+
+describe('attempt to fix tests', () => {
+  test('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running') // to check if this is being run
+    if (process.env.SHOULD_ALWAYS_PASS) {
+      expect(1 + 2).to.equal(3)
+    } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+      // We need the last attempt to fail for the exit code to be 1
+      if (numAttempt++ % 2 === 1) {
+        expect(1 + 2).to.equal(4)
+      } else {
+        expect(1 + 2).to.equal(3)
+      }
+    } else {
+      expect(1 + 2).to.equal(4)
+    }
+  })
+})

integration-tests/cucumber/cucumber.spec.js

@@ -49,7 +49,10 @@ const {
   TEST_MANAGEMENT_IS_DISABLED,
   DD_CAPABILITIES_TEST_IMPACT_ANALYSIS,
   DD_CAPABILITIES_EARLY_FLAKE_DETECTION,
-  DD_CAPABILITIES_AUTO_TEST_RETRIES
+  DD_CAPABILITIES_AUTO_TEST_RETRIES,
+  TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
+  TEST_HAS_FAILED_ALL_RETRIES,
+  TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED
 } = require('../../packages/dd-trace/src/plugins/util/test')
 const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
 
@@ -2031,6 +2034,229 @@ versions.forEach(version => {
     })
 
     context('test management', () => {
+      context('attempt to fix', () => {
+        beforeEach(() => {
+          receiver.setTestManagementTests({
+            cucumber: {
+              suites: {
+                'ci-visibility/features-test-management/attempt-to-fix.feature': {
+                  tests: {
+                    'Say attempt to fix': {
+                      properties: {
+                        attempt_to_fix: true
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          })
+        })
+
+        const getTestAssertions = ({
+          isAttemptToFix,
+          isQuarantined,
+          isDisabled,
+          shouldAlwaysPass,
+          shouldFailSometimes
+        }) =>
+          receiver
+            .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+              const events = payloads.flatMap(({ payload }) => payload.events)
+              const tests = events.filter(event => event.type === 'test').map(event => event.content)
+              const testSession = events.find(event => event.type === 'test_session_end').content
+
+              if (isAttemptToFix) {
+                assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true')
+              } else {
+                assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED)
+              }
+
+              const retriedTests = tests.filter(
+                test => test.meta[TEST_NAME] === 'Say attempt to fix'
+              )
+
+              if (isAttemptToFix) {
+                // 3 retries + 1 initial run
+                assert.equal(retriedTests.length, 4)
+              } else {
+                assert.equal(retriedTests.length, 1)
+              }
+
+              for (let i = 0; i < retriedTests.length; i++) {
+                const isFirstAttempt = i === 0
+                const isLastAttempt = i === retriedTests.length - 1
+                const test = retriedTests[i]
+
+                assert.equal(
+                  test.resource,
+                  'ci-visibility/features-test-management/attempt-to-fix.feature.Say attempt to fix'
+                )
+
+                if (isDisabled) {
+                  assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true')
+                } else if (isQuarantined) {
+                  assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true')
+                } else {
+                  assert.notProperty(test.meta, TEST_MANAGEMENT_IS_DISABLED)
+                  assert.notProperty(test.meta, TEST_MANAGEMENT_IS_QUARANTINED)
+                }
+
+                if (isAttemptToFix) {
+                  assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true')
+                  if (!isFirstAttempt) {
+                    assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
+                    assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
+                  }
+                  if (isLastAttempt) {
+                    if (shouldFailSometimes) {
+                      assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED)
+                      assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES)
+                    } else if (shouldAlwaysPass) {
+                      assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true')
+                    } else {
+                      assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true')
+                    }
+                  }
+                } else {
+                  assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX)
+                  assert.notProperty(test.meta, TEST_IS_RETRY)
+                  assert.notProperty(test.meta, TEST_RETRY_REASON)
+                }
+              }
+            })
+
+        const runTest = (done, {
+          isAttemptToFix,
+          isQuarantined,
+          isDisabled,
+          extraEnvVars,
+          shouldAlwaysPass,
+          shouldFailSometimes
+        } = {}) => {
+          const testAssertions = getTestAssertions({
+            isAttemptToFix,
+            isQuarantined,
+            isDisabled,
+            shouldAlwaysPass,
+            shouldFailSometimes
+          })
+          let stdout = ''
+
+          childProcess = exec(
+            './node_modules/.bin/cucumber-js ci-visibility/features-test-management/attempt-to-fix.feature',
+            {
+              cwd,
+              env: {
+                ...getCiVisAgentlessConfig(receiver.port),
+                ...extraEnvVars,
+                ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}),
+                ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {})
+              },
+              stdio: 'inherit'
+            }
+          )
+
+          childProcess.stdout.on('data', (data) => {
+            stdout += data.toString()
+          })
+
+          childProcess.on('exit', exitCode => {
+            testAssertions.then(() => {
+              assert.include(stdout, 'I am running')
+              if (isQuarantined || isDisabled || shouldAlwaysPass) {
+                assert.equal(exitCode, 0)
+              } else {
+                assert.equal(exitCode, 1)
+              }
+              done()
+            }).catch(done)
+          })
+        }
+
+        it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runTest(done, { isAttemptToFix: true })
+        })
+
+        it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runTest(done, { isAttemptToFix: true, shouldAlwaysPass: true })
+        })
+
+        it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runTest(done, { isAttemptToFix: true, shouldFailSometimes: true })
+        })
+
+        it('does not attempt to fix tests if test management is not enabled', (done) => {
+          receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } })
+
+          runTest(done)
+        })
+
+        it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+
+          runTest(done, {
+            extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' }
+          })
+        })
+
+        it('does not fail retry if a test is quarantined', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+          receiver.setTestManagementTests({
+            cucumber: {
+              suites: {
+                'ci-visibility/features-test-management/attempt-to-fix.feature': {
+                  tests: {
+                    'Say attempt to fix': {
+                      properties: {
+                        attempt_to_fix: true,
+                        quarantined: true
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          })
+
+          runTest(done, {
+            isAttemptToFix: true,
+            isQuarantined: true
+          })
+        })
+
+        it('does not fail retry if a test is disabled', (done) => {
+          receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
+          receiver.setTestManagementTests({
+            cucumber: {
+              suites: {
+                'ci-visibility/features-test-management/attempt-to-fix.feature': {
+                  tests: {
+                    'Say attempt to fix': {
+                      properties: {
+                        attempt_to_fix: true,
+                        disabled: true
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          })
+
+          runTest(done, {
+            isAttemptToFix: true,
+            isDisabled: true
+          })
+        })
+      })
+
       context('disabled', () => {
         beforeEach(() => {
           receiver.setTestManagementTests({