diff --git a/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature b/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature new file mode 100644 index 00000000000..98ac42a8213 --- /dev/null +++ b/integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature @@ -0,0 +1,4 @@ +Feature: Attempt to fix + Scenario: Say attempt to fix + When the greeter says attempt to fix + Then I should have heard "attempt to fix" diff --git a/integration-tests/ci-visibility/features-test-management/support/steps.js b/integration-tests/ci-visibility/features-test-management/support/steps.js index e01c21e968a..67a2ed51361 100644 --- a/integration-tests/ci-visibility/features-test-management/support/steps.js +++ b/integration-tests/ci-visibility/features-test-management/support/steps.js @@ -1,6 +1,8 @@ const assert = require('assert') const { When, Then } = require('@cucumber/cucumber') +let numAttempt = 0 + Then('I should have heard {string}', function (expectedResponse) { if (this.whatIHeard === 'quarantine') { assert.equal(this.whatIHeard, 'fail') @@ -21,3 +23,16 @@ When('the greeter says disabled', function () { // expected to fail if not disabled this.whatIHeard = 'disabld' }) + +When('the greeter says attempt to fix', function () { + // eslint-disable-next-line no-console + console.log('I am running') // just to assert whether this is running + // expected to fail + if (process.env.SHOULD_ALWAYS_PASS) { + this.whatIHeard = 'attempt to fix' + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + this.whatIHeard = numAttempt++ % 2 === 0 ? 'attempt to fix' : 'attempt to fx' + } else { + this.whatIHeard = 'attempt to fx' + } +}) diff --git a/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js b/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js new file mode 100644 index 00000000000..f235d10f549 --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js @@ -0,0 +1,27 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test.describe('attempt to fix', () => { + test('should attempt to fix failed test', async ({ page }) => { + let textToAssert + + if (process.env.SHOULD_ALWAYS_PASS) { + textToAssert = 'Hello World' + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + // can't use numAttempt++ because we're running in parallel + if (Number(process.env.TEST_WORKER_INDEX) % 2 === 0) { + throw new Error('Hello Warld') + } + textToAssert = 'Hello World' + } else { + textToAssert = 'Hello Warld' + } + + await expect(page.locator('.hello-world')).toHaveText([ + textToAssert + ]) + }) +}) diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js new file mode 100644 index 00000000000..be05f47fd50 --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js @@ -0,0 +1,21 @@ +const { expect } = require('chai') + +let numAttempts = 0 + +describe('attempt to fix tests', () => { + it('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running when attempt to fix') // to check if this is being run + if (process.env.SHOULD_ALWAYS_PASS) { + expect(1 + 2).to.equal(3) + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + if (numAttempts++ % 2 === 0) { + expect(1 + 2).to.equal(3) + } else { + expect(1 + 2).to.equal(4) + } + } else { + expect(1 + 2).to.equal(4) + } + }) +}) diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js new file mode 100644 index 00000000000..053d1d62eb0 --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') + +describe('attempt to fix tests 2', () => { + it('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running when attempt to fix 2') // to check if this is being run + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs b/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs new file mode 100644 index 00000000000..4fe4ba6cacc --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs @@ -0,0 +1,22 @@ +import { describe, test, expect } from 'vitest' + +let numAttempt = 0 + +describe('attempt to fix tests', () => { + test('can attempt to fix a test', () => { + // eslint-disable-next-line no-console + console.log('I am running') // to check if this is being run + if (process.env.SHOULD_ALWAYS_PASS) { + expect(1 + 2).to.equal(3) + } else if (process.env.SHOULD_FAIL_SOMETIMES) { + // We need the last attempt to fail for the exit code to be 1 + if (numAttempt++ % 2 === 1) { + expect(1 + 2).to.equal(4) + } else { + expect(1 + 2).to.equal(3) + } + } else { + expect(1 + 2).to.equal(4) + } + }) +}) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 2359577b9ea..092dcbe3af7 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -49,7 +49,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -2031,6 +2034,229 @@ versions.forEach(version => { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'Say attempt to fix' + ) + + if (isAttemptToFix) { + // 3 retries + 1 initial run + assert.equal(retriedTests.length, 4) + } else { + assert.equal(retriedTests.length, 1) + } + + for (let i = 0; i < retriedTests.length; i++) { + const isFirstAttempt = i === 0 + const isLastAttempt = i === retriedTests.length - 1 + const test = retriedTests[i] + + assert.equal( + test.resource, + 'ci-visibility/features-test-management/attempt-to-fix.feature.Say attempt to fix' + ) + + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } else if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_DISABLED) + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + + if (isAttemptToFix) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (!isFirstAttempt) { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + if (isLastAttempt) { + if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + } + }) + + const runTest = (done, { + isAttemptToFix, + isQuarantined, + isDisabled, + extraEnvVars, + shouldAlwaysPass, + shouldFailSometimes + } = {}) => { + const testAssertions = getTestAssertions({ + isAttemptToFix, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) + let stdout = '' + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-test-management/attempt-to-fix.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + ...extraEnvVars, + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data.toString() + }) + + childProcess.on('exit', exitCode => { + testAssertions.then(() => { + assert.include(stdout, 'I am running') + if (isQuarantined || isDisabled || shouldAlwaysPass) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runTest(done, { + extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } + }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runTest(done, { + isAttemptToFix: true, + isQuarantined: true + }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cucumber: { + suites: { + 'ci-visibility/features-test-management/attempt-to-fix.feature': { + tests: { + 'Say attempt to fix': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runTest(done, { + isAttemptToFix: true, + isDisabled: true + }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 76958fbdac0..1ab83ca8064 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -44,7 +44,11 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_NAME, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1775,6 +1779,245 @@ moduleTypes.forEach(({ }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + cypress: { + suites: { + 'cypress/e2e/attempt-to-fix.js': { + tests: { + 'attempt to fix is attempt to fix': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'cypress/e2e/attempt-to-fix.js.attempt to fix is attempt to fix' + ] + ) + + const attemptToFixTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix is attempt to fix' + ) + + if (isAttemptToFix) { + assert.equal(attemptToFixTests.length, 4) + } else { + assert.equal(attemptToFixTests.length, 1) + } + + for (let i = attemptToFixTests.length - 1; i >= 0; i--) { + const test = attemptToFixTests[i] + if (!isAttemptToFix) { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + assert.notPropertyVal(test.meta, TEST_STATUS, 'skip') + } + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + assert.notPropertyVal(test.meta, TEST_STATUS, 'skip') + } + + const isLastAttempt = i === attemptToFixTests.length - 1 + const isFirstAttempt = i === 0 + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } else { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + if (isLastAttempt) { + if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } else if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled, + extraEnvVars = {} + } = {}) => { + const testAssertionsPromise = getTestAssertions({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) + + const { + NODE_OPTIONS, + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/attempt-to-fix.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + ...extraEnvVars, + ...(shouldAlwaysPass ? { CYPRESS_SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { CYPRESS_SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (shouldAlwaysPass) { + assert.equal(exitCode, 0) + } else { + // TODO: we need to figure out how to trick cypress into returning exit code 0 + // even if there are failed tests + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + /** + * TODO: + * The spec says that quarantined tests that are not attempted to fix should be run and their result ignored. + * Cypress will skip the test instead. + * + * When a test is quarantined and attempted to fix, the spec is to run the test and ignore its result. + * Cypress will run the test, but it won't ignore its result. + */ + it('can mark tests as quarantined and tests are not skipped', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cypress: { + suites: { + 'cypress/e2e/attempt-to-fix.js': { + tests: { + 'attempt to fix is attempt to fix': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isQuarantined: true }) + }) + + /** + * TODO: + * When a test is disabled and attempted to fix, the spec is to run the test and ignore its result. + * Cypress will run the test, but it won't ignore its result. + */ + it('can mark tests as disabled and tests are not skipped', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + cypress: { + suites: { + 'cypress/e2e/attempt-to-fix.js': { + tests: { + 'attempt to fix is attempt to fix': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/cypress/e2e/attempt-to-fix.js b/integration-tests/cypress/e2e/attempt-to-fix.js new file mode 100644 index 00000000000..9c638c4b6dd --- /dev/null +++ b/integration-tests/cypress/e2e/attempt-to-fix.js @@ -0,0 +1,20 @@ +/* eslint-disable */ + +let numAttempt = 0 + +function getTextToAssert () { + if (Cypress.env('SHOULD_ALWAYS_PASS')) { + return 'Hello World' + } else if (Cypress.env('SHOULD_FAIL_SOMETIMES')) { + return numAttempt++ % 2 === 0 ? 'Hello World' : 'Hello Warld' + } + return 'Hello Warld' +} + +describe('attempt to fix', () => { + it('is attempt to fix', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', getTextToAssert()) + }) +}) diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index 06708e793eb..f2352437d2a 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -46,7 +46,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2942,6 +2945,257 @@ describe('jest CommonJS', () => { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + isParallel, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-management/test-attempt-to-fix-1.js.attempt to fix tests can attempt to fix a test' + ] + ) + + if (isParallel) { + // Parallel mode in jest requires more than a single test suite + // Here we check that the second test suite is actually running, + // so we can be sure that parallel mode is on + const parallelTestName = 'ci-visibility/test-management/test-attempt-to-fix-2.js.' + + 'attempt to fix tests 2 can attempt to fix a test' + assert.includeMembers(resourceNames, [parallelTestName]) + } + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + if (!isAttemptToFix) { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + + if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } + + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } + + const isFirstAttempt = i === 0 + const isLastAttempt = i === retriedTests.length - 1 + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } else { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + + if (isLastAttempt) { + if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } else if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptToFix, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes, + extraEnvVars = {}, + isParallel = false + } = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions({ + isAttemptToFix, + isParallel, + isQuarantined, + isDisabled, + shouldAlwaysPass, + shouldFailSometimes + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test-management/test-attempt-to-fix-1', + SHOULD_CHECK_RESULTS: '1', + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}), + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.stderr.on('data', (chunk) => { + stdout += chunk.toString() + }) + + childProcess.stdout.on('data', (chunk) => { + stdout += chunk.toString() + }) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running when attempt to fix') + if (isQuarantined || shouldAlwaysPass || isDisabled) { + // even though a test fails, the exit code is 0 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isQuarantined: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + jest: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true }) + }) + + it('can attempt to fix in parallel mode', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest( + done, + { + isAttemptToFix: true, + isParallel: true, + extraEnvVars: { + // we need to run more than 1 suite for parallel mode to kick in + TESTS_TO_RUN: 'test-management/test-attempt-to-fix', + RUN_IN_PARALLEL: true + } + } + ) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index 6c96d2fb136..c96d99953c4 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -48,7 +48,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2566,6 +2569,227 @@ describe('mocha CommonJS', function () { }) context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-management/test-attempt-to-fix-1.js.attempt to fix tests can attempt to fix a test' + ] + ) + + const retriedTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < retriedTests.length; i++) { + const test = retriedTests[i] + const isFirstAttempt = i === 0 + const isLastAttempt = i === retriedTests.length - 1 + if (!isAttemptToFix) { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } else { + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + } + + if (isQuarantined) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } + + if (isDisabled) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } + + if (isLastAttempt) { + if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + } + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled, + extraEnvVars = {} + } = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions({ + isAttemptToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantined, + isDisabled + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-management/test-attempt-to-fix-1.js' + ]), + SHOULD_CHECK_RESULTS: '1', + ...extraEnvVars, + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data + }) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running when attempt to fix') + if (shouldAlwaysPass || isQuarantined || isDisabled) { + // even though a test fails, the exit code is 0 because the test is quarantined or disabled + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isQuarantined: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-1.js': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 0ed40cdfe7d..347ae5f980d 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -33,7 +33,11 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_NAME, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -898,6 +902,228 @@ versions.forEach((version) => { if (version === 'latest') { context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isDisabled, + isQuarantined + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptingToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const attemptedToFixTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix should attempt to fix failed test' + ) + + if (isAttemptingToFix) { + assert.equal(attemptedToFixTests.length, 4) + } else { + assert.equal(attemptedToFixTests.length, 1) + } + + if (isDisabled) { + const numDisabledTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_DISABLED] === 'true' + ).length + assert.equal(numDisabledTests, attemptedToFixTests.length) + } + + if (isQuarantined) { + const numQuarantinedTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_QUARANTINED] === 'true' + ).length + assert.equal(numQuarantinedTests, attemptedToFixTests.length) + } + + // Retried tests are in randomly order, so we just count number of tests + const countAttemptToFixTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true' + ).length + + const countRetriedAttemptToFixTests = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true' && + test.meta[TEST_IS_RETRY] === 'true' && + test.meta[TEST_RETRY_REASON] === 'attempt_to_fix' + ).length + + const testsMarkedAsFailedAllRetries = attemptedToFixTests.filter(test => + test.meta[TEST_HAS_FAILED_ALL_RETRIES] === 'true' + ).length + + const testsMarkedAsPassedAllRetries = attemptedToFixTests.filter(test => + test.meta[TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED] === 'true' + ).length + + if (isAttemptingToFix) { + assert.equal(countAttemptToFixTests, attemptedToFixTests.length) + assert.equal(countRetriedAttemptToFixTests, attemptedToFixTests.length - 1) + if (shouldAlwaysPass) { + assert.equal(testsMarkedAsFailedAllRetries, 0) + assert.equal(testsMarkedAsPassedAllRetries, 1) + } else if (shouldFailSometimes) { + assert.equal(testsMarkedAsFailedAllRetries, 0) + assert.equal(testsMarkedAsPassedAllRetries, 0) + } else { // always fail + assert.equal(testsMarkedAsFailedAllRetries, 1) + assert.equal(testsMarkedAsPassedAllRetries, 0) + } + } else { + assert.equal(countAttemptToFixTests, 0) + assert.equal(countRetriedAttemptToFixTests, 0) + assert.equal(testsMarkedAsFailedAllRetries, 0) + assert.equal(testsMarkedAsPassedAllRetries, 0) + } + }) + + const runAttemptToFixTest = (done, { + isAttemptingToFix, + isQuarantined, + extraEnvVars, + shouldAlwaysPass, + shouldFailSometimes, + isDisabled + } = {}) => { + const testAssertionsPromise = getTestAssertions({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isDisabled, + isQuarantined + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js attempt-to-fix-test.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-test-management', + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}), + ...extraEnvVars + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (isQuarantined || isDisabled || shouldAlwaysPass) { + // even though a test fails, the exit code is 0 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isQuarantined: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + playwright: { + suites: { + 'attempt-to-fix-test.js': { + tests: { + 'attempt to fix should attempt to fix failed test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isDisabled: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index a4d68ce87b8..3e26a1c9d03 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -37,7 +37,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -1341,6 +1344,217 @@ versions.forEach((version) => { if (version === 'latest') { context('test management', () => { + context('attempt to fix', () => { + beforeEach(() => { + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = ({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantining, + isDisabling + }) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isAttemptingToFix) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs.attempt to fix tests can attempt to fix a test' + ] + ) + + const attemptedToFixTests = tests.filter( + test => test.meta[TEST_NAME] === 'attempt to fix tests can attempt to fix a test' + ) + + for (let i = 0; i < attemptedToFixTests.length; i++) { + const isFirstAttempt = i === 0 + const isLastAttempt = i === attemptedToFixTests.length - 1 + const test = attemptedToFixTests[i] + if (isQuarantining) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else if (isDisabling) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_DISABLED, 'true') + } + + if (isAttemptingToFix) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (isFirstAttempt) { + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + continue + } + assert.propertyVal(test.meta, TEST_IS_RETRY, 'true') + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix') + if (isLastAttempt) { + if (shouldAlwaysPass) { + assert.propertyVal(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } else if (shouldFailSometimes) { + assert.notProperty(test.meta, TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED) + assert.notProperty(test.meta, TEST_HAS_FAILED_ALL_RETRIES) + } else { + assert.propertyVal(test.meta, TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + } + } else { + assert.notProperty(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX) + assert.notProperty(test.meta, TEST_IS_RETRY) + assert.notProperty(test.meta, TEST_RETRY_REASON) + } + } + }) + + const runAttemptToFixTest = (done, { + isAttemptingToFix, + shouldAlwaysPass, + isQuarantining, + shouldFailSometimes, + isDisabling, + extraEnvVars = {} + } = {}) => { + let stdout = '' + const testAssertionsPromise = getTestAssertions({ + isAttemptingToFix, + shouldAlwaysPass, + shouldFailSometimes, + isQuarantining, + isDisabling + }) + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/test-attempt-to-fix*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init --no-warnings', + ...extraEnvVars, + ...(shouldAlwaysPass ? { SHOULD_ALWAYS_PASS: '1' } : {}), + ...(shouldFailSometimes ? { SHOULD_FAIL_SOMETIMES: '1' } : {}) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (data) => { + stdout += data + }) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + assert.include(stdout, 'I am running') + if (shouldAlwaysPass || (isAttemptingToFix && isQuarantining) || (isAttemptingToFix && isDisabling)) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can attempt to fix and mark last attempt as failed if every attempt fails', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true }) + }) + + it('can attempt to fix and mark last attempt as passed if every attempt passes', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldAlwaysPass: true }) + }) + + it('can attempt to fix and not mark last attempt if attempts both pass and fail', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, shouldFailSometimes: true }) + }) + + it('does not attempt to fix tests if test management is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done) + }) + + it('does not enable attempt to fix tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + + runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) + }) + + it('does not fail retry if a test is quarantined', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + quarantined: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isQuarantining: true }) + }) + + it('does not fail retry if a test is disabled', (done) => { + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) + receiver.setTestManagementTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': { + tests: { + 'attempt to fix tests can attempt to fix a test': { + properties: { + attempt_to_fix: true, + disabled: true + } + } + } + } + } + } + }) + + runAttemptToFixTest(done, { isAttemptingToFix: true, isDisabling: true }) + }) + }) + context('disabled', () => { beforeEach(() => { receiver.setTestManagementTests({ @@ -1428,7 +1642,7 @@ versions.forEach((version) => { assert.equal(exitCode, 1) } done() - }) + }).catch(done) }) } @@ -1541,7 +1755,7 @@ versions.forEach((version) => { assert.equal(exitCode, 1) } done() - }) + }).catch(done) }) } diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index ce82c268e3f..bc7bc55b932 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -73,6 +73,7 @@ let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false +let testManagementAttemptToFixRetries = 0 let testManagementTests = {} let numTestRetries = 0 let knownTests = [] @@ -121,10 +122,10 @@ function isNewTest (testSuite, testName) { } function getTestProperties (testSuite, testName) { - const { disabled, quarantined } = + const { attempt_to_fix: attemptToFix, disabled, quarantined } = testManagementTests?.cucumber?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { disabled, quarantined } + return { attemptToFix, disabled, quarantined } } function getTestStatusFromRetries (testStatuses) { @@ -303,22 +304,42 @@ function wrapRun (pl, isLatestVersion) { } let isNew = false let isEfdRetry = false + let isAttemptToFix = false + let isAttemptToFixRetry = false + let hasFailedAllRetries = false + let hasPassedAllRetries = false let isDisabled = false let isQuarantined = false - if (isKnownTestsEnabled && status !== 'skip') { - const numRetries = numRetriesByPickleId.get(this.pickle.id) - isNew = numRetries !== undefined - isEfdRetry = numRetries > 0 - } if (isTestManagementTestsEnabled) { const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) const testProperties = getTestProperties(testSuitePath, this.pickle.name) + const numRetries = numRetriesByPickleId.get(this.pickle.id) + isAttemptToFix = testProperties.attemptToFix + isAttemptToFixRetry = isAttemptToFix && numRetries > 0 isDisabled = testProperties.disabled - if (!isDisabled) { - isQuarantined = testProperties.quarantined + isQuarantined = testProperties.quarantined + + if (isAttemptToFixRetry) { + const statuses = lastStatusByPickleId.get(this.pickle.id) + if (statuses.length === testManagementAttemptToFixRetries + 1) { + const { pass, fail } = statuses.reduce((acc, status) => { + acc[status]++ + return acc + }, { pass: 0, fail: 0 }) + hasFailedAllRetries = fail === testManagementAttemptToFixRetries + 1 + hasPassedAllRetries = pass === testManagementAttemptToFixRetries + 1 + } } } + + if (isKnownTestsEnabled && status !== 'skip' && !isAttemptToFix) { + const numRetries = numRetriesByPickleId.get(this.pickle.id) + + isNew = numRetries !== undefined + isEfdRetry = numRetries > 0 + } + const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) const error = getErrorFromCucumberResult(result) @@ -334,6 +355,10 @@ function wrapRun (pl, isLatestVersion) { isNew, isEfdRetry, isFlakyRetry: numAttempt > 0, + isAttemptToFix, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAllRetries, isDisabled, isQuarantined }) @@ -426,6 +451,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount isKnownTestsEnabled = configurationResponse.libraryConfig?.isKnownTestsEnabled isTestManagementTestsEnabled = configurationResponse.libraryConfig?.isTestManagementEnabled + testManagementAttemptToFixRetries = configurationResponse.libraryConfig?.testManagementAttemptToFixRetries if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) @@ -576,22 +602,25 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } let isNew = false + let isAttemptToFix = false let isDisabled = false let isQuarantined = false - if (isKnownTestsEnabled) { - isNew = isNewTest(testSuitePath, pickle.name) - if (isNew) { - numRetriesByPickleId.set(pickle.id, 0) - } - } if (isTestManagementTestsEnabled) { const testProperties = getTestProperties(testSuitePath, pickle.name) + isAttemptToFix = testProperties.attemptToFix isDisabled = testProperties.disabled - if (isDisabled) { + isQuarantined = testProperties.quarantined + // If attempt to fix is enabled, we run even if the test is disabled + if (!isAttemptToFix && isDisabled) { this.options.dryRun = true - } else { - isQuarantined = testProperties.quarantined + } + } + + if (isKnownTestsEnabled && !isAttemptToFix) { + isNew = isNewTest(testSuitePath, pickle.name) + if (isNew) { + numRetriesByPickleId.set(pickle.id, 0) } } // TODO: for >=11 we could use `runTestCaseResult` instead of accumulating results in `lastStatusByPickleId` @@ -599,6 +628,15 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa const testStatuses = lastStatusByPickleId.get(pickle.id) const lastTestStatus = testStatuses[testStatuses.length - 1] + + // New tests should not be marked as attempt to fix, so EFD + Attempt to fix should not be enabled at the same time + if (isAttemptToFix && lastTestStatus !== 'skip') { + for (let retryIndex = 0; retryIndex < testManagementAttemptToFixRetries; retryIndex++) { + numRetriesByPickleId.set(pickle.id, retryIndex + 1) + runTestCaseResult = await runTestCaseFunction.apply(this, arguments) + } + } + // If it's a new test and it hasn't been skipped, we run it again if (isEarlyFlakeDetectionEnabled && lastTestStatus !== 'skip' && isNew) { for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { @@ -608,7 +646,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } let testStatus = lastTestStatus let shouldBePassedByEFD = false - let shouldBePassedByQuarantine = false + let shouldBePassedByTestManagement = false if (isNew && isEarlyFlakeDetectionEnabled) { /** * If Early Flake Detection (EFD) is enabled the logic is as follows: @@ -625,9 +663,9 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } } - if (isTestManagementTestsEnabled && isQuarantined) { + if (isTestManagementTestsEnabled && (isDisabled || isQuarantined)) { this.success = true - shouldBePassedByQuarantine = true + shouldBePassedByTestManagement = true } if (!pickleResultByFile[testFileAbsolutePath]) { @@ -661,8 +699,8 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa return shouldBePassedByEFD } - if (isNewerCucumberVersion && isTestManagementTestsEnabled && isQuarantined) { - return shouldBePassedByQuarantine + if (isNewerCucumberVersion && isTestManagementTestsEnabled && (isQuarantined || isDisabled)) { + return shouldBePassedByTestManagement } return runTestCaseResult diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index 1f16f98fe73..e53251aa3d9 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -13,7 +13,9 @@ const { addEfdStringToTestName, removeEfdStringFromTestName, getIsFaultyEarlyFlakeDetection, - JEST_WORKER_LOGS_PAYLOAD_CODE + JEST_WORKER_LOGS_PAYLOAD_CODE, + addAttemptToFixStringToTestName, + removeAttemptToFixStringFromTestName } = require('../../dd-trace/src/plugins/util/test') const { getFormattedJestTestParameters, @@ -73,6 +75,7 @@ let hasFilteredSkippableSuites = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false let testManagementTests = {} +let testManagementAttemptToFixRetries = 0 const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -80,6 +83,7 @@ const asyncResources = new WeakMap() const originalTestFns = new WeakMap() const retriedTestsToNumAttempts = new Map() const newTestsTestStatuses = new Map() +const attemptToFixRetriedTestsStatuses = new Map() const BREAKPOINT_HIT_GRACE_PERIOD_MS = 200 @@ -110,7 +114,7 @@ function getTestEnvironmentOptions (config) { return {} } -function getEfdStats (testStatuses) { +function getTestStats (testStatuses) { return testStatuses.reduce((acc, testStatus) => { acc[testStatus]++ return acc @@ -169,6 +173,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (this.isTestManagementTestsEnabled) { try { const hasTestManagementTests = !!testManagementTests.jest + testManagementAttemptToFixRetries = this.testEnvironmentOptions._ddTestManagementAttemptToFixRetries this.testManagementTestsForThisSuite = hasTestManagementTests ? this.getTestManagementTestsForSuite(testManagementTests.jest.suites?.[this.testSuite]?.tests) : this.getTestManagementTestsForSuite(this.testEnvironmentOptions._ddTestManagementTests) @@ -213,9 +218,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (this.testManagementTestsForThisSuite) { return this.testManagementTestsForThisSuite } - // TODO - ADD ATTEMPT_TO_FIX tests if (!testManagementTests) { return { + attemptToFix: [], disabled: [], quarantined: [] } @@ -228,14 +233,19 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } const result = { + attemptToFix: [], disabled: [], quarantined: [] } Object.entries(testManagementTestsForSuite).forEach(([testName, { properties }]) => { + if (properties?.attempt_to_fix) { + result.attemptToFix.push(testName) + } if (properties?.disabled) { result.disabled.push(testName) - } else if (properties?.quarantined) { + } + if (properties?.quarantined) { result.quarantined.push(testName) } }) @@ -243,11 +253,30 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { return result } - // Add the `add_test` event we don't have the test object yet, so + // Generic function to handle test retries + retryTest (testName, retryCount, addRetryStringToTestName, retryType, event) { + // Retrying snapshots has proven to be problematic, so we'll skip them for now + // We'll still detect new tests, but we won't retry them. + // TODO: do not bail out of retrying tests for the whole test suite + if (this.getHasSnapshotTests()) { + log.warn(`${retryType} is disabled for suites with snapshots`) + return + } + + for (let retryIndex = 0; retryIndex < retryCount; retryIndex++) { + if (this.global.test) { + this.global.test(addRetryStringToTestName(testName, retryIndex), event.fn, event.timeout) + } else { + log.error(`${retryType} could not retry test because global.test is undefined`) + } + } + } + + // At the `add_test` event we don't have the test object yet, so we can't use it getTestNameFromAddTestEvent (event, state) { const describeSuffix = getJestTestName(state.currentDescribeBlock) const fullTestName = describeSuffix ? `${describeSuffix} ${event.testName}` : event.testName - return removeEfdStringFromTestName(fullTestName) + return removeAttemptToFixStringFromTestName(removeEfdStringFromTestName(fullTestName)) } async handleTestEvent (event, state) { @@ -273,15 +302,31 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (event.name === 'test_start') { let isNewTest = false let numEfdRetry = null + let numOfAttemptsToFixRetries = null const testParameters = getTestParametersString(this.nameToParams, event.test.name) // Async resource for this test is created here // It is used later on by the test_done handler const asyncResource = new AsyncResource('bound-anonymous-fn') asyncResources.set(event.test, asyncResource) const testName = getJestTestName(event.test) + const originalTestName = removeEfdStringFromTestName(removeAttemptToFixStringFromTestName(testName)) + + let isAttemptToFix = false + let isDisabled = false + let isQuarantined = false + if (this.isTestManagementTestsEnabled) { + isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(originalTestName) + isDisabled = this.testManagementTestsForThisSuite?.disabled?.includes(originalTestName) + isQuarantined = this.testManagementTestsForThisSuite?.quarantined?.includes(originalTestName) + if (isAttemptToFix) { + numOfAttemptsToFixRetries = retriedTestsToNumAttempts.get(originalTestName) + retriedTestsToNumAttempts.set(originalTestName, numOfAttemptsToFixRetries + 1) + } else if (isDisabled) { + event.test.mode = 'skip' + } + } if (this.isKnownTestsEnabled) { - const originalTestName = removeEfdStringFromTestName(testName) isNewTest = retriedTestsToNumAttempts.has(originalTestName) if (isNewTest) { numEfdRetry = retriedTestsToNumAttempts.get(originalTestName) @@ -289,16 +334,10 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } - if (this.isTestManagementTestsEnabled) { - const isDisabled = this.testManagementTestsForThisSuite?.disabled?.includes(testName) - if (isDisabled) { - event.test.mode = 'skip' - } - } const isJestRetry = event.test?.invocations > 1 asyncResource.runInAsyncScope(() => { testStartCh.publish({ - name: removeEfdStringFromTestName(testName), + name: originalTestName, suite: this.testSuite, testSourceFile: this.testSourceFile, displayName: this.displayName, @@ -306,34 +345,46 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { frameworkVersion: jestVersion, isNew: isNewTest, isEfdRetry: numEfdRetry > 0, - isJestRetry + isAttemptToFix, + isAttemptToFixRetry: numOfAttemptsToFixRetries > 0, + isJestRetry, + isDisabled, + isQuarantined }) originalTestFns.set(event.test, event.test.fn) event.test.fn = asyncResource.bind(event.test.fn) }) } + if (event.name === 'add_test') { + const originalTestName = this.getTestNameFromAddTestEvent(event, state) + + const isSkipped = event.mode === 'todo' || event.mode === 'skip' + if (this.isTestManagementTestsEnabled) { + const isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(originalTestName) + if (isAttemptToFix && !isSkipped && !retriedTestsToNumAttempts.has(originalTestName)) { + retriedTestsToNumAttempts.set(originalTestName, 0) + this.retryTest( + event.testName, + testManagementAttemptToFixRetries, + addAttemptToFixStringToTestName, + 'Test Management (Attempt to Fix)', + event + ) + } + } if (this.isKnownTestsEnabled) { - const testName = this.getTestNameFromAddTestEvent(event, state) - const isNew = !this.knownTestsForThisSuite?.includes(testName) - const isSkipped = event.mode === 'todo' || event.mode === 'skip' - if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(testName)) { - retriedTestsToNumAttempts.set(testName, 0) + const isNew = !this.knownTestsForThisSuite?.includes(originalTestName) + if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(originalTestName)) { + retriedTestsToNumAttempts.set(originalTestName, 0) if (this.isEarlyFlakeDetectionEnabled) { - // Retrying snapshots has proven to be problematic, so we'll skip them for now - // We'll still detect new tests, but we won't retry them. - // TODO: do not bail out of EFD with the whole test suite - if (this.getHasSnapshotTests()) { - log.warn('Early flake detection is disabled for suites with snapshots') - return - } - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { - if (this.global.test) { - this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) - } else { - log.error('Early flake detection could not retry test because global.test is undefined') - } - } + this.retryTest( + event.testName, + earlyFlakeDetectionNumRetries, + addEfdStringToTestName, + 'Early flake detection', + event + ) } } } @@ -346,6 +397,32 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { // restore in case it is retried event.test.fn = originalTestFns.get(event.test) + let attemptToFixPassed = false + let failedAllTests = false + if (this.isTestManagementTestsEnabled) { + const testName = getJestTestName(event.test) + const originalTestName = removeAttemptToFixStringFromTestName(testName) + const isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(originalTestName) + if (isAttemptToFix) { + if (attemptToFixRetriedTestsStatuses.has(originalTestName)) { + attemptToFixRetriedTestsStatuses.get(originalTestName).push(status) + } else { + attemptToFixRetriedTestsStatuses.set(originalTestName, [status]) + } + const testStatuses = attemptToFixRetriedTestsStatuses.get(originalTestName) + // Check if this is the last attempt to fix. + // If it is, we'll set the failedAllTests flag to true if all the tests failed + // If all tests passed, we'll set the attemptToFixPassed flag to true + if (testStatuses.length === testManagementAttemptToFixRetries + 1) { + if (testStatuses.every(status => status === 'fail')) { + failedAllTests = true + } else if (testStatuses.every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + } + } + // We'll store the test statuses of the retries if (this.isKnownTestsEnabled) { const testName = getJestTestName(event.test) @@ -359,12 +436,6 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } } - let isQuarantined = false - - if (this.isTestManagementTestsEnabled) { - const testName = getJestTestName(event.test) - isQuarantined = this.testManagementTestsForThisSuite?.quarantined?.includes(testName) - } const promises = {} const numRetries = this.global[RETRY_TIMES] @@ -399,7 +470,8 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { testFinishCh.publish({ status, testStartLine: getTestLineStart(event.test.asyncError, this.testSuite), - isQuarantined + attemptToFixPassed, + failedAllTests }) }) @@ -552,6 +624,7 @@ function cliWrapper (cli, jestVersion) { earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (err) { log.error('Jest library configuration error', err) @@ -710,7 +783,7 @@ function cliWrapper (cli, jestVersion) { if (isEarlyFlakeDetectionEnabled) { let numFailedTestsToIgnore = 0 for (const testStatuses of newTestsTestStatuses.values()) { - const { pass, fail } = getEfdStats(testStatuses) + const { pass, fail } = getTestStats(testStatuses) if (pass > 0) { // as long as one passes, we'll consider the test passed numFailedTestsToIgnore += fail } @@ -725,29 +798,41 @@ function cliWrapper (cli, jestVersion) { const failedTests = result .results .testResults.flatMap(({ testResults, testFilePath: testSuiteAbsolutePath }) => ( - testResults.map(({ fullName: testName, status }) => ({ testName, testSuiteAbsolutePath, status })) + testResults.map(({ fullName: testName, status }) => ( + { testName, testSuiteAbsolutePath, status } + )) )) .filter(({ status }) => status === 'failed') let numFailedQuarantinedTests = 0 + let numFailedQuarantinedOrDisabledAttemptedToFixTests = 0 for (const { testName, testSuiteAbsolutePath } of failedTests) { const testSuite = getTestSuitePath(testSuiteAbsolutePath, result.globalConfig.rootDir) - const isQuarantined = testManagementTests + const originalName = removeAttemptToFixStringFromTestName(testName) + const testManagementTest = testManagementTests ?.jest ?.suites ?.[testSuite] ?.tests - ?.[testName] + ?.[originalName] ?.properties - ?.quarantined - if (isQuarantined) { + // This uses `attempt_to_fix` because this is always the main process and it's not formatted in camelCase + if (testManagementTest?.attempt_to_fix && (testManagementTest?.quarantined || testManagementTest?.disabled)) { + numFailedQuarantinedOrDisabledAttemptedToFixTests++ + } else if (testManagementTest?.quarantined) { numFailedQuarantinedTests++ } } // If every test that failed was quarantined, we'll consider the suite passed - if (numFailedQuarantinedTests !== 0 && result.results.numFailedTests === numFailedQuarantinedTests) { + // Note that if a test is attempted to fix, + // it's considered quarantined both if it's disabled and if it's quarantined (it'll run but its status is ignored) + if ( + (numFailedQuarantinedOrDisabledAttemptedToFixTests !== 0 || numFailedQuarantinedTests !== 0) && + result.results.numFailedTests === + numFailedQuarantinedTests + numFailedQuarantinedOrDisabledAttemptedToFixTests + ) { result.results.success = true } } @@ -947,6 +1032,7 @@ addHook({ _ddIsKnownTestsEnabled, _ddIsTestManagementTestsEnabled, _ddTestManagementTests, + _ddTestManagementAttemptToFixRetries, ...restOfTestEnvironmentOptions } = testEnvironmentOptions diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 6589e3ccacb..70427ac05dc 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -30,7 +30,9 @@ const { newTests, testsQuarantined, getTestFullName, - getRunTestsWrapper + getRunTestsWrapper, + testsAttemptToFix, + testsStatuses } = require('./utils') require('./common') @@ -138,16 +140,26 @@ function getOnEndHandler (isParallel) { } } + // We substract the errors of attempt to fix tests (quarantined or disabled) from the total number of failures // We subtract the errors from quarantined tests from the total number of failures if (config.isTestManagementTestsEnabled) { let numFailedQuarantinedTests = 0 + let numFailedRetriedQuarantinedOrDisabledTests = 0 + for (const test of testsAttemptToFix) { + const testName = getTestFullName(test) + const testProperties = getTestProperties(test, config.testManagementTests) + if (isTestFailed(test) && (testProperties.isQuarantined || testProperties.isDisabled)) { + const numFailedTests = testsStatuses.get(testName).filter(status => status === 'fail').length + numFailedRetriedQuarantinedOrDisabledTests += numFailedTests + } + } for (const test of testsQuarantined) { if (isTestFailed(test)) { numFailedQuarantinedTests++ } } - this.stats.failures -= numFailedQuarantinedTests - this.failures -= numFailedQuarantinedTests + this.stats.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests + this.failures -= numFailedQuarantinedTests + numFailedRetriedQuarantinedOrDisabledTests } if (status === 'fail') { @@ -193,6 +205,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { if (err) { config.testManagementTests = {} config.isTestManagementTestsEnabled = false + config.testManagementAttemptToFixRetries = 0 } else { config.testManagementTests = receivedTestManagementTests } @@ -260,6 +273,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold config.isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled config.isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + config.testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries // ITR and auto test retries are not supported in parallel mode yet config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled @@ -401,7 +415,7 @@ addHook({ this.on('test', getOnTestHandler(true)) - this.on('test end', getOnTestEndHandler()) + this.on('test end', getOnTestEndHandler(config)) this.on('retry', getOnTestRetryHandler()) @@ -637,6 +651,8 @@ addHook({ if (config.isTestManagementTestsEnabled) { const testSuiteTestManagementTests = config.testManagementTests?.mocha?.suites?.[testPath] || {} newWorkerArgs._ddIsTestManagementTestsEnabled = true + // TODO: attempt to fix does not work in parallel mode yet + // newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries newWorkerArgs._ddTestManagementTests = { mocha: { suites: { diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index ce33d1cf7c4..27f8d4faf4b 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -3,7 +3,9 @@ const { getTestSuitePath, removeEfdStringFromTestName, - addEfdStringToTestName + addEfdStringToTestName, + addAttemptToFixStringToTestName, + removeAttemptToFixStringFromTestName } = require('../../../dd-trace/src/plugins/util/test') const { channel, AsyncResource } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') @@ -26,7 +28,9 @@ const testToStartLine = new WeakMap() const testFileToSuiteAr = new Map() const wrappedFunctions = new WeakSet() const newTests = {} +const testsAttemptToFix = new Set() const testsQuarantined = new Set() +const testsStatuses = new Map() function getAfterEachHooks (testOrHook) { const hooks = [] @@ -44,10 +48,10 @@ function getTestProperties (test, testManagementTests) { const testSuite = getTestSuitePath(test.file, process.cwd()) const testName = test.fullTitle() - const { disabled: isDisabled, quarantined: isQuarantined } = + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = testManagementTests?.mocha?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } function isNewTest (test, knownTests) { @@ -57,15 +61,18 @@ function isNewTest (test, knownTests) { return !testsForSuite.includes(testName) } -function retryTest (test, earlyFlakeDetectionNumRetries) { +function retryTest (test, numRetries, modifyTestName, tags) { const originalTestName = test.title const suite = test.parent - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) { const clonedTest = test.clone() - clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) + clonedTest.title = modifyTestName(originalTestName, retryIndex + 1) suite.addTest(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true + tags.forEach(tag => { + if (tag) { + clonedTest[tag] = true + } + }) } } @@ -102,7 +109,10 @@ function getIsLastRetry (test) { } function getTestFullName (test) { - return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` + const testName = removeEfdStringFromTestName( + removeAttemptToFixStringFromTestName(test.fullTitle()) + ) + return `mocha.${getTestSuitePath(test.file, process.cwd())}.${testName}` } function getTestStatus (test) { @@ -195,12 +205,15 @@ function getOnTestHandler (isMain) { title, _ddIsNew: isNew, _ddIsEfdRetry: isEfdRetry, + _ddIsAttemptToFix: isAttemptToFix, _ddIsDisabled: isDisabled, _ddIsQuarantined: isQuarantined } = test + const testName = removeEfdStringFromTestName(removeAttemptToFixStringFromTestName(test.fullTitle())) + const testInfo = { - testName: test.fullTitle(), + testName, testSuiteAbsolutePath, title, testStartLine @@ -212,6 +225,7 @@ function getOnTestHandler (isMain) { testInfo.isNew = isNew testInfo.isEfdRetry = isEfdRetry + testInfo.isAttemptToFix = isAttemptToFix testInfo.isDisabled = isDisabled testInfo.isQuarantined = isQuarantined // We want to store the result of the new tests @@ -224,7 +238,7 @@ function getOnTestHandler (isMain) { } } - if (isDisabled) { + if (!isAttemptToFix && isDisabled) { test.pending = true } @@ -234,7 +248,7 @@ function getOnTestHandler (isMain) { } } -function getOnTestEndHandler () { +function getOnTestEndHandler (config) { return async function (test) { const asyncResource = getTestAsyncResource(test) const status = getTestStatus(test) @@ -249,13 +263,40 @@ function getOnTestEndHandler () { }) } + let hasFailedAllRetries = false + let attemptToFixPassed = false + + const testName = getTestFullName(test) + + if (!testsStatuses.get(testName)) { + testsStatuses.set(testName, [status]) + } else { + testsStatuses.get(testName).push(status) + } + const testStatuses = testsStatuses.get(testName) + + const isLastAttempt = testStatuses.length === config.testManagementAttemptToFixRetries + 1 + + if (test._ddIsAttemptToFix && isLastAttempt) { + if (testStatuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } else if (testStatuses.every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + + const isAttemptToFixRetry = test._ddIsAttemptToFix && testStatuses.length > 1 + // if there are afterEach to be run, we don't finish the test yet if (asyncResource && !getAfterEachHooks(test).length) { asyncResource.runInAsyncScope(() => { testFinishCh.publish({ status, hasBeenRetried: isMochaRetry(test), - isLastRetry: getIsLastRetry(test) + isLastRetry: getIsLastRetry(test), + hasFailedAllRetries, + attemptToFixPassed, + isAttemptToFixRetry }) }) } @@ -374,33 +415,50 @@ function getOnPendingHandler () { } } -// Hook to add retries to tests if EFD is enabled +// Hook to add retries to tests if Test Management or EFD is enabled function getRunTestsWrapper (runTests, config) { - return function (suite, fn) { + return function (suite) { + if (config.isTestManagementTestsEnabled) { + suite.tests.forEach((test) => { + const { isAttemptToFix, isDisabled, isQuarantined } = getTestProperties(test, config.testManagementTests) + if (isAttemptToFix && !test.isPending()) { + test._ddIsAttemptToFix = true + test._ddIsDisabled = isDisabled + test._ddIsQuarantined = isQuarantined + // This is needed to know afterwards which ones have been retried to ignore its result + testsAttemptToFix.add(test) + retryTest( + test, + config.testManagementAttemptToFixRetries, + addAttemptToFixStringToTestName, + ['_ddIsAttemptToFix', isDisabled && '_ddIsDisabled', isQuarantined && '_ddIsQuarantined'] + ) + } else if (isDisabled) { + test._ddIsDisabled = true + } else if (isQuarantined) { + testsQuarantined.add(test) + test._ddIsQuarantined = true + } + }) + } + if (config.isKnownTestsEnabled) { // by the time we reach `this.on('test')`, it is too late. We need to add retries here suite.tests.forEach(test => { if (!test.isPending() && isNewTest(test, config.knownTests)) { test._ddIsNew = true if (config.isEarlyFlakeDetectionEnabled) { - retryTest(test, config.earlyFlakeDetectionNumRetries) + retryTest( + test, + config.earlyFlakeDetectionNumRetries, + addEfdStringToTestName, + ['_ddIsNew', '_ddIsEfdRetry'] + ) } } }) } - if (config.isTestManagementTestsEnabled) { - suite.tests.forEach(test => { - const { isDisabled, isQuarantined } = getTestProperties(test, config.testManagementTests) - if (isDisabled) { - test._ddIsDisabled = true - } else if (isQuarantined) { - testsQuarantined.add(test) - test._ddIsQuarantined = true - } - }) - } - return runTests.apply(this, arguments) } } @@ -408,7 +466,6 @@ function getRunTestsWrapper (runTests, config) { module.exports = { isNewTest, getTestProperties, - retryTest, getSuitesByTestFile, isMochaRetry, getTestFullName, @@ -427,5 +484,7 @@ module.exports = { testFileToSuiteAr, getRunTestsWrapper, newTests, - testsQuarantined + testsQuarantined, + testsAttemptToFix, + testsStatuses } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index d456150036f..c8f58109f0e 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -36,6 +36,8 @@ addHook({ } if (this.options._ddIsTestManagementTestsEnabled) { config.isTestManagementTestsEnabled = true + // TODO: attempt to fix does not work in parallel mode yet + // config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries config.testManagementTests = this.options._ddTestManagementTests delete this.options._ddIsTestManagementTestsEnabled delete this.options._ddTestManagementTests @@ -64,7 +66,7 @@ addHook({ }) this.on('test', getOnTestHandler(false)) - this.on('test end', getOnTestEndHandler()) + this.on('test end', getOnTestEndHandler(config)) // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted this.on('hook end', getOnHookEndHandler()) diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index f56f88a565f..0df5addc442 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -22,6 +22,7 @@ const testToAr = new WeakMap() const testSuiteToAr = new Map() const testSuiteToTestStatuses = new Map() const testSuiteToErrors = new Map() +const testsToTestStatuses = new Map() const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') let applyRepeatEachIndex = null @@ -43,7 +44,9 @@ let isFlakyTestRetriesEnabled = false let flakyTestRetriesCount = 0 let knownTests = {} let isTestManagementTestsEnabled = false +let testManagementAttemptToFixRetries = 0 let testManagementTests = {} +const quarantinedOrDisabledTestsAttemptToFix = [] let rootDir = '' const MINIMUM_SUPPORTED_VERSION_RANGE_EFD = '>=1.38.0' @@ -51,10 +54,9 @@ function getTestProperties (test) { const testName = getTestFullname(test) const testSuite = getTestSuitePath(test._requireFile, rootDir) - const { disabled, quarantined } = + const { attempt_to_fix: attemptToFix, disabled, quarantined } = testManagementTests?.playwright?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - - return { disabled, quarantined } + return { attemptToFix, disabled, quarantined } } function isNewTest (test) { @@ -73,16 +75,19 @@ function getSuiteType (test, type) { } // Copy of Suite#_deepClone but with a function to filter tests -function deepCloneSuite (suite, filterTest) { +function deepCloneSuite (suite, filterTest, tags = []) { const copy = suite._clone() for (const entry of suite._entries) { if (entry.constructor.name === 'Suite') { - copy._addSuite(deepCloneSuite(entry, filterTest)) + copy._addSuite(deepCloneSuite(entry, filterTest, tags)) } else { if (filterTest(entry)) { const copiedTest = entry._clone() - copiedTest._ddIsNew = true - copiedTest._ddIsEfdRetry = true + tags.forEach(tag => { + if (tag) { + copiedTest[tag] = true + } + }) copy._addTest(copiedTest) } } @@ -276,6 +281,11 @@ function testBeginHandler (test, browserName) { }) } + // We disable retries by default if attemptToFix is true + if (getTestProperties(test).attemptToFix) { + test.retries = 0 + } + const testAsyncResource = new AsyncResource('bound-anonymous-fn') testToAr.set(test, testAsyncResource) testAsyncResource.runInAsyncScope(() => { @@ -288,7 +298,6 @@ function testBeginHandler (test, browserName) { }) }) } - function testEndHandler (test, annotations, testStatus, error, isTimeout) { let annotationTags if (annotations.length) { @@ -305,6 +314,27 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { return } + const testFullName = getTestFullname(test) + const testFqn = `${testSuiteAbsolutePath} ${testFullName}` + const testStatuses = testsToTestStatuses.get(testFqn) || [] + + if (testStatuses.length === 0) { + testsToTestStatuses.set(testFqn, [testStatus]) + } else { + testStatuses.push(testStatus) + } + + let hasFailedAllRetries = false + let hasPassedAttemptToFixRetries = false + + if (testStatuses.length === testManagementAttemptToFixRetries + 1) { + if (testStatuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } else if (testStatuses.every(status => status === 'pass')) { + hasPassedAttemptToFixRetries = true + } + } + const testResult = results[results.length - 1] const testAsyncResource = testToAr.get(test) testAsyncResource.runInAsyncScope(() => { @@ -315,8 +345,12 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { error, extraTags: annotationTags, isNew: test._ddIsNew, + isAttemptToFix: test._ddIsAttemptToFix, + isAttemptToFixRetry: test._ddIsAttemptToFixRetry, isQuarantined: test._ddIsQuarantined, - isEfdRetry: test._ddIsEfdRetry + isEfdRetry: test._ddIsEfdRetry, + hasFailedAllRetries, + hasPassedAttemptToFixRetries }) }) @@ -338,7 +372,6 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { // Last test, we finish the suite if (!remainingTestsByFile[testSuiteAbsolutePath].length) { const testStatuses = testSuiteToTestStatuses.get(testSuiteAbsolutePath) - let testSuiteStatus = 'pass' if (testStatuses.some(status => status === 'fail')) { testSuiteStatus = 'fail' @@ -445,6 +478,7 @@ function runnerHook (runnerExport, playwrightVersion) { isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (e) { isEarlyFlakeDetectionEnabled = false @@ -485,7 +519,10 @@ function runnerHook (runnerExport, playwrightVersion) { const projects = getProjectsFromRunner(this) - if (isFlakyTestRetriesEnabled && flakyTestRetriesCount > 0) { + const shouldSetRetries = isFlakyTestRetriesEnabled && + flakyTestRetriesCount > 0 && + !isTestManagementTestsEnabled + if (shouldSetRetries) { projects.forEach(project => { if (project.retries === 0) { // Only if it hasn't been set by the user project.retries = flakyTestRetriesCount @@ -493,7 +530,7 @@ function runnerHook (runnerExport, playwrightVersion) { }) } - const runAllTestsReturn = await runAllTests.apply(this, arguments) + let runAllTestsReturn = await runAllTests.apply(this, arguments) Object.values(remainingTestsByFile).forEach(tests => { // `tests` should normally be empty, but if it isn't, @@ -508,6 +545,26 @@ function runnerHook (runnerExport, playwrightVersion) { const sessionStatus = runAllTestsReturn.status || runAllTestsReturn + if (isTestManagementTestsEnabled && sessionStatus === 'failed') { + let totalFailedTestCount = 0 + let totalAttemptToFixFailedTestCount = 0 + + for (const testStatuses of testsToTestStatuses.values()) { + totalFailedTestCount += testStatuses.filter(status => status === 'fail').length + } + + for (const test of quarantinedOrDisabledTestsAttemptToFix) { + const fullname = getTestFullname(test) + const fqn = `${test._requireFile} ${fullname}` + const testStatuses = testsToTestStatuses.get(fqn) + totalAttemptToFixFailedTestCount += testStatuses.filter(status => status === 'fail').length + } + + if (totalFailedTestCount === totalAttemptToFixFailedTestCount) { + runAllTestsReturn = 'passed' + } + } + const flushWait = new Promise(resolve => { onDone = resolve }) @@ -608,9 +665,28 @@ addHook({ const testProperties = getTestProperties(test) if (testProperties.disabled) { test._ddIsDisabled = true - test.expectedStatus = 'skipped' } else if (testProperties.quarantined) { test._ddIsQuarantined = true + } + if (testProperties.attemptToFix) { + test._ddIsAttemptToFix = true + const fileSuite = getSuiteType(test, 'file') + const projectSuite = getSuiteType(test, 'project') + const isAttemptToFix = test => getTestProperties(test).attemptToFix + for (let repeatEachIndex = 1; repeatEachIndex <= testManagementAttemptToFixRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, isAttemptToFix, [ + testProperties.disabled && '_ddIsDisabled', + testProperties.quarantined && '_ddIsQuarantined', + '_ddIsAttemptToFix', + '_ddIsAttemptToFixRetry' + ]) + applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) + projectSuite._addSuite(copyFileSuite) + } + if (testProperties.disabled || testProperties.quarantined) { + quarantinedOrDisabledTestsAttemptToFix.push(test) + } + } else if (testProperties.disabled || testProperties.quarantined) { test.expectedStatus = 'skipped' } } @@ -619,18 +695,22 @@ addHook({ if (isKnownTestsEnabled) { const newTests = allTests.filter(isNewTest) - newTests.forEach(newTest => { + for (const newTest of newTests) { + // No need to filter out attempt to fix tests here because attempt to fix tests are never new newTest._ddIsNew = true if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped') { const fileSuite = getSuiteType(newTest, 'file') const projectSuite = getSuiteType(newTest, 'project') - for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { - const copyFileSuite = deepCloneSuite(fileSuite, isNewTest) + for (let repeatEachIndex = 1; repeatEachIndex <= earlyFlakeDetectionNumRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, isNewTest, [ + '_ddIsNew', + '_ddIsEfdRetry' + ]) applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) projectSuite._addSuite(copyFileSuite) } } - }) + } } return rootSuite diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index 4ac4062b46c..b3bf9809c4c 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -9,6 +9,7 @@ const testPassCh = channel('ci:vitest:test:pass') const testErrorCh = channel('ci:vitest:test:error') const testSkipCh = channel('ci:vitest:test:skip') const isNewTestCh = channel('ci:vitest:test:is-new') +const isAttemptToFixCh = channel('ci:vitest:test:is-attempt-to-fix') const isDisabledCh = channel('ci:vitest:test:is-disabled') const isQuarantinedCh = channel('ci:vitest:test:is-quarantined') @@ -30,7 +31,9 @@ const taskToStatuses = new WeakMap() const newTasks = new WeakSet() const disabledTasks = new WeakSet() const quarantinedTasks = new WeakSet() +const attemptToFixTasks = new WeakSet() let isRetryReasonEfd = false +let isRetryReasonAttemptToFix = false const switchedStatuses = new WeakSet() const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -53,6 +56,7 @@ function getProvidedContext () { _ddEarlyFlakeDetectionNumRetries: numRepeats, _ddIsKnownTestsEnabled: isKnownTestsEnabled, _ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled, + _ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries, _ddTestManagementTests: testManagementTests, _ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled } = globalThis.__vitest_worker__.providedContext @@ -64,6 +68,7 @@ function getProvidedContext () { numRepeats, isKnownTestsEnabled, isTestManagementTestsEnabled, + testManagementAttemptToFixRetries, testManagementTests, isFlakyTestRetriesEnabled } @@ -76,6 +81,7 @@ function getProvidedContext () { numRepeats: 0, isKnownTestsEnabled: false, isTestManagementTestsEnabled: false, + testManagementAttemptToFixRetries: 0, testManagementTests: {} } } @@ -176,6 +182,7 @@ function getSortWrapper (sort) { let isEarlyFlakeDetectionFaulty = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false + let testManagementAttemptToFixRetries = 0 let isDiEnabled = false let knownTests = {} let testManagementTests = {} @@ -190,6 +197,7 @@ function getSortWrapper (sort) { isDiEnabled = libraryConfig.isDiEnabled isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries } } catch (e) { isFlakyTestRetriesEnabled = false @@ -262,6 +270,7 @@ function getSortWrapper (sort) { try { const workspaceProject = this.ctx.getCoreWorkspaceProject() workspaceProject._provided._ddIsTestManagementTestsEnabled = isTestManagementTestsEnabled + workspaceProject._provided._ddTestManagementAttemptToFixRetries = testManagementAttemptToFixRetries workspaceProject._provided._ddTestManagementTests = testManagementTests } catch (e) { log.warn('Could not send test management tests to workers so Test Management will not work.') @@ -353,10 +362,24 @@ addHook({ isKnownTestsEnabled, numRepeats, isTestManagementTestsEnabled, + testManagementAttemptToFixRetries, testManagementTests } = getProvidedContext() if (isTestManagementTestsEnabled) { + isAttemptToFixCh.publish({ + testManagementTests, + testSuiteAbsolutePath: task.file.filepath, + testName, + onDone: (isAttemptToFix) => { + if (isAttemptToFix) { + isRetryReasonAttemptToFix = task.repeats !== testManagementAttemptToFixRetries + task.repeats = testManagementAttemptToFixRetries + attemptToFixTasks.add(task) + taskToStatuses.set(task, []) + } + } + }) isDisabledCh.publish({ testManagementTests, testSuiteAbsolutePath: task.file.filepath, @@ -364,7 +387,10 @@ addHook({ onDone: (isTestDisabled) => { if (isTestDisabled) { disabledTasks.add(task) - task.mode = 'skip' + if (!attemptToFixTasks.has(task)) { + // we only actually skip if the test is not being attempted to be fixed + task.mode = 'skip' + } } } }) @@ -376,7 +402,7 @@ addHook({ testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isNew) => { - if (isNew) { + if (isNew && !attemptToFixTasks.has(task)) { if (isEarlyFlakeDetectionEnabled) { isRetryReasonEfd = task.repeats !== numRepeats task.repeats = numRepeats @@ -396,19 +422,28 @@ addHook({ shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => function (task) { const { isEarlyFlakeDetectionEnabled, isTestManagementTestsEnabled } = getProvidedContext() - if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task)) { - const statuses = taskToStatuses.get(task) - // If the test has passed at least once, we consider it passed - if (statuses.includes('pass')) { + if (isTestManagementTestsEnabled) { + const isAttemptingToFix = attemptToFixTasks.has(task) + const isDisabled = disabledTasks.has(task) + const isQuarantined = quarantinedTasks.has(task) + + if (isAttemptingToFix && (isDisabled || isQuarantined)) { if (task.result.state === 'fail') { switchedStatuses.add(task) } task.result.state = 'pass' + } else if (isQuarantined) { + task.result.state = 'pass' } } - if (isTestManagementTestsEnabled) { - if (quarantinedTasks.has(task)) { + if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + // If the test has passed at least once, we consider it passed + if (statuses.includes('pass')) { + if (task.result.state === 'fail') { + switchedStatuses.add(task) + } task.result.state = 'pass' } } @@ -481,6 +516,8 @@ addHook({ } const lastExecutionStatus = task.result.state + const shouldFlipStatus = isEarlyFlakeDetectionEnabled || attemptToFixTasks.has(task) + const statuses = taskToStatuses.get(task) // These clauses handle task.repeats, whether EFD is enabled or not // The only thing that EFD does is to forcefully pass the test if it has passed at least once @@ -501,15 +538,21 @@ addHook({ testPassCh.publish({ task }) }) } - if (isEarlyFlakeDetectionEnabled) { - const statuses = taskToStatuses.get(task) + if (shouldFlipStatus) { statuses.push(lastExecutionStatus) // If we don't "reset" the result.state to "pass", once a repetition fails, // vitest will always consider the test as failed, so we can't read the actual status + // This means that we change vitest's behavior: + // if the last attempt passes, vitest would consider the test as failed + // but after this change, it will consider the test as passed task.result.state = 'pass' } } } else if (numRepetition === task.repeats) { + if (shouldFlipStatus) { + statuses.push(lastExecutionStatus) + } + const asyncResource = taskToAsync.get(task) if (lastExecutionStatus === 'fail') { const testError = task.result?.errors?.[0] @@ -532,8 +575,11 @@ addHook({ testSuiteAbsolutePath: task.file.filepath, isRetry: numAttempt > 0 || numRepetition > 0, isRetryReasonEfd, + isRetryReasonAttemptToFix: isRetryReasonAttemptToFix && numRepetition > 0, isNew, mightHitProbe: isDiEnabled && numAttempt > 0, + isAttemptToFix: attemptToFixTasks.has(task), + isDisabled: disabledTasks.has(task), isQuarantined }) }) @@ -548,6 +594,8 @@ addHook({ } const result = await onAfterTryTask.apply(this, arguments) + const { testManagementAttemptToFixRetries } = getProvidedContext() + const status = getVitestTestStatus(task, retryCount) const asyncResource = taskToAsync.get(task) @@ -557,10 +605,18 @@ addHook({ await waitForHitProbe() } + let attemptToFixPassed = false + if (attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + if (statuses.length === testManagementAttemptToFixRetries && statuses.every(status => status === 'pass')) { + attemptToFixPassed = true + } + } + if (asyncResource) { // We don't finish here because the test might fail in a later hook (afterEach) asyncResource.runInAsyncScope(() => { - testFinishTimeCh.publish({ status, task }) + testFinishTimeCh.publish({ status, task, attemptToFixPassed }) }) } @@ -715,11 +771,19 @@ addHook({ testError = errors[0] } + let hasFailedAllRetries = false + if (attemptToFixTasks.has(task)) { + const statuses = taskToStatuses.get(task) + if (statuses.every(status => status === 'fail')) { + hasFailedAllRetries = true + } + } + if (testAsyncResource) { const isRetry = task.result?.retryCount > 0 // `duration` is the duration of all the retries, so it can't be used if there are retries testAsyncResource.runInAsyncScope(() => { - testErrorCh.publish({ duration: !isRetry ? duration : undefined, error: testError }) + testErrorCh.publish({ duration: !isRetry ? duration : undefined, error: testError, hasFailedAllRetries }) }) } if (errors?.length) { diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index f7ff7e4c2e9..37d58e4fcec 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -30,7 +30,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -328,6 +331,10 @@ class CucumberPlugin extends CiPlugin { isNew, isEfdRetry, isFlakyRetry, + isAttemptToFix, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAllRetries, isDisabled, isQuarantined }) => { @@ -358,6 +365,22 @@ class CucumberPlugin extends CiPlugin { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + + if (isAttemptToFix) { + span.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + } + + if (isAttemptToFixRetry) { + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + if (hasPassedAllRetries) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + } + if (isDisabled) { span.setTag(TEST_MANAGEMENT_IS_DISABLED, 'true') } diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 8767b6b9016..092db83a901 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -39,7 +39,10 @@ const { TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, DD_CAPABILITIES_AUTO_TEST_RETRIES, - DD_CAPABILITIES_TEST_IMPACT_ANALYSIS + DD_CAPABILITIES_TEST_IMPACT_ANALYSIS, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') @@ -64,7 +67,8 @@ const { GIT_COMMIT_SHA, GIT_BRANCH, CI_PROVIDER_NAME, - CI_WORKSPACE_PATH + CI_WORKSPACE_PATH, + GIT_COMMIT_MESSAGE } = require('../../dd-trace/src/plugins/util/tags') const { OS_VERSION, @@ -201,7 +205,8 @@ class CypressPlugin { [RUNTIME_VERSION]: runtimeVersion, [GIT_BRANCH]: branch, [CI_PROVIDER_NAME]: ciProviderName, - [CI_WORKSPACE_PATH]: repositoryRoot + [CI_WORKSPACE_PATH]: repositoryRoot, + [GIT_COMMIT_MESSAGE]: commitMessage } = this.testEnvironmentMetadata this.repositoryRoot = repositoryRoot @@ -217,9 +222,11 @@ class CypressPlugin { runtimeName, runtimeVersion, branch, - testLevel: 'test' + testLevel: 'test', + commitMessage } this.finishedTestsByFile = {} + this.testStatuses = {} this.isTestsSkipped = false this.isSuitesSkippingEnabled = false @@ -233,6 +240,8 @@ class CypressPlugin { this.hasUnskippableSuites = false this.unskippableSuites = [] this.knownTests = [] + this.isTestManagementTestsEnabled = false + this.testManagementAttemptToFixRetries = 0 } // Init function returns a promise that resolves with the Cypress configuration @@ -261,7 +270,8 @@ class CypressPlugin { isFlakyTestRetriesEnabled, flakyTestRetriesCount, isKnownTestsEnabled, - isTestManagementEnabled + isTestManagementEnabled, + testManagementAttemptToFixRetries } } = libraryConfigurationResponse this.isSuitesSkippingEnabled = isSuitesSkippingEnabled @@ -273,17 +283,22 @@ class CypressPlugin { this.cypressConfig.retries.runMode = flakyTestRetriesCount } this.isTestManagementTestsEnabled = isTestManagementEnabled + this.testManagementAttemptToFixRetries = testManagementAttemptToFixRetries } return this.cypressConfig }) return this.libraryConfigurationPromise } + getTestSuiteProperties (testSuite) { + return this.testManagementTests?.cypress?.suites?.[testSuite]?.tests || {} + } + getTestProperties (testSuite, testName) { - const { disabled: isDisabled, quarantined: isQuarantined } = - this.testManagementTests?.cypress?.suites?.[testSuite]?.tests?.[testName]?.properties || {} + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = + this.getTestSuiteProperties(testSuite)?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } getTestSuiteSpan ({ testSuite, testSuiteAbsolutePath }) { @@ -312,7 +327,7 @@ class CypressPlugin { }) } - getTestSpan ({ testName, testSuite, isUnskippable, isForcedToRun, testSourceFile }) { + getTestSpan ({ testName, testSuite, isUnskippable, isForcedToRun, testSourceFile, isDisabled, isQuarantined }) { const testSuiteTags = { [TEST_COMMAND]: this.command, [TEST_COMMAND]: this.command, @@ -357,6 +372,14 @@ class CypressPlugin { testSpanMetadata[TEST_ITR_FORCED_RUN] = 'true' } + if (isDisabled) { + testSpanMetadata[TEST_MANAGEMENT_IS_DISABLED] = 'true' + } + + if (isQuarantined) { + testSpanMetadata[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' + } + this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'test', { hasCodeOwners: !!codeOwners }) return this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test`, { @@ -691,7 +714,10 @@ class CypressPlugin { isEarlyFlakeDetectionEnabled: this.isEarlyFlakeDetectionEnabled, knownTestsForSuite: this.knownTestsByTestSuite?.[testSuite] || [], earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries, - isKnownTestsEnabled: this.isKnownTestsEnabled + isKnownTestsEnabled: this.isKnownTestsEnabled, + isTestManagementEnabled: this.isTestManagementTestsEnabled, + testManagementAttemptToFixRetries: this.testManagementAttemptToFixRetries, + testManagementTests: this.getTestSuiteProperties(testSuite) } if (this.testSuiteSpan) { @@ -707,8 +733,7 @@ class CypressPlugin { }) const isUnskippable = this.unskippableSuites.includes(testSuite) const isForcedToRun = shouldSkip && isUnskippable - const { isDisabled, isQuarantined } = this.getTestProperties(testSuite, testName) - + const { isAttemptToFix, isDisabled, isQuarantined } = this.getTestProperties(testSuite, testName) // skip test if (shouldSkip && !isUnskippable) { this.skippedTests.push(test) @@ -718,7 +743,7 @@ class CypressPlugin { // TODO: I haven't found a way to trick cypress into ignoring a test // The way we'll implement quarantine in cypress is by skipping the test altogether - if (isDisabled || isQuarantined) { + if (!isAttemptToFix && (isDisabled || isQuarantined)) { return { shouldSkip: true } } @@ -727,7 +752,9 @@ class CypressPlugin { testName, testSuite, isUnskippable, - isForcedToRun + isForcedToRun, + isDisabled, + isQuarantined }) } @@ -747,7 +774,8 @@ class CypressPlugin { testSuiteAbsolutePath, testName, isNew, - isEfdRetry + isEfdRetry, + isAttemptToFix } = test if (coverage && this.isCodeCoverageEnabled && this.tracer._tracer._exporter?.exportCoverage) { const coverageFiles = getCoveredFilenamesFromCoverage(coverage) @@ -770,6 +798,14 @@ class CypressPlugin { const testStatus = CYPRESS_STATUS_TO_TEST_STATUS[state] this.activeTestSpan.setTag(TEST_STATUS, testStatus) + // Save the test status to know if it has passed all retries + if (!this.testStatuses[testName]) { + this.testStatuses[testName] = [testStatus] + } else { + this.testStatuses[testName].push(testStatus) + } + const testStatuses = this.testStatuses[testName] + if (error) { this.activeTestSpan.setTag('error', error) } @@ -786,6 +822,22 @@ class CypressPlugin { this.activeTestSpan.setTag(TEST_RETRY_REASON, 'efd') } } + if (isAttemptToFix) { + this.activeTestSpan.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + if (testStatuses.length > 1) { + this.activeTestSpan.setTag(TEST_IS_RETRY, 'true') + this.activeTestSpan.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + } + const isLastAttempt = testStatuses.length === this.testManagementAttemptToFixRetries + 1 + if (isLastAttempt) { + if (testStatuses.every(status => status === 'fail')) { + this.activeTestSpan.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } else if (testStatuses.every(status => status === 'pass')) { + this.activeTestSpan.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + } + } + const finishedTest = { testName, testStatus, diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index 749a25d7f66..668e146a2f5 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -4,6 +4,9 @@ let isKnownTestsEnabled = false let knownTestsForSuite = [] let suiteTests = [] let earlyFlakeDetectionNumRetries = 0 +let isTestManagementEnabled = false +let testManagementAttemptToFixRetries = 0 +let testManagementTests = {} // We need to grab the original window as soon as possible, // in case the test changes the origin. If the test does change the origin, // any call to `cy.window()` will result in a cross origin error. @@ -23,31 +26,53 @@ function isNewTest (test) { return !knownTestsForSuite.includes(test.fullTitle()) } -function retryTest (test, suiteTests) { - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { +function getTestProperties (testName) { + // We neeed to do it in this way because of compatibility with older versions as '?' is not supported in older versions of Cypress + const properties = testManagementTests[testName] && testManagementTests[testName].properties || {}; + + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = properties; + + return { isAttemptToFix, isDisabled, isQuarantined }; +} + +function retryTest (test, suiteTests, numRetries, tags) { + for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) { const clonedTest = test.clone() // TODO: signal in framework logs that this is a retry. // TODO: Change it so these tests are allowed to fail. // TODO: figure out if reported duration is skewed. suiteTests.unshift(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true + tags.forEach(tag => { + clonedTest[tag] = true + }) } } const oldRunTests = Cypress.mocha.getRunner().runTests Cypress.mocha.getRunner().runTests = function (suite, fn) { - if (!isKnownTestsEnabled) { + if (!isKnownTestsEnabled && !isTestManagementEnabled) { return oldRunTests.apply(this, arguments) } // We copy the new tests at the beginning of the suite run (runTests), so that they're run // multiple times. suite.tests.forEach(test => { - if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { - test._ddIsNew = true - if (isEarlyFlakeDetectionEnabled) { - retryTest(test, suite.tests) + const testName = test.fullTitle() + + const { isAttemptToFix } = getTestProperties(testName) + + if (isTestManagementEnabled) { + if (isAttemptToFix && !test.isPending()) { + test._ddIsAttemptToFix = true + retryTest(test, suite.tests, testManagementAttemptToFixRetries, ['_ddIsAttemptToFix']) + } + } + if (isKnownTestsEnabled) { + if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { + test._ddIsNew = true + if (isEarlyFlakeDetectionEnabled && !isAttemptToFix) { + retryTest(test, suite.tests, earlyFlakeDetectionNumRetries, ['_ddIsNew', '_ddIsEfdRetry']) + } } } }) @@ -80,6 +105,9 @@ before(function () { isKnownTestsEnabled = suiteConfig.isKnownTestsEnabled knownTestsForSuite = suiteConfig.knownTestsForSuite earlyFlakeDetectionNumRetries = suiteConfig.earlyFlakeDetectionNumRetries + isTestManagementEnabled = suiteConfig.isTestManagementEnabled + testManagementAttemptToFixRetries = suiteConfig.testManagementAttemptToFixRetries + testManagementTests = suiteConfig.testManagementTests } }) }) @@ -104,7 +132,8 @@ afterEach(function () { state: currentTest.state, error: currentTest.err, isNew: currentTest._ddIsNew, - isEfdRetry: currentTest._ddIsEfdRetry + isEfdRetry: currentTest._ddIsEfdRetry, + isAttemptToFix: currentTest._ddIsAttemptToFix } try { testInfo.testSourceLine = Cypress.mocha.getRunner().currentRunnable.invocationDetails.line diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 1e231443772..e7b36d17341 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -27,7 +27,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') @@ -177,6 +180,7 @@ class JestPlugin extends CiPlugin { config._ddRepositoryRoot = this.repositoryRoot config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false config._ddIsTestManagementTestsEnabled = this.libraryConfig?.isTestManagementEnabled ?? false + config._ddTestManagementAttemptToFixRetries = this.libraryConfig?.testManagementAttemptToFixRetries ?? 0 config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false config._ddIsKnownTestsEnabled = this.libraryConfig?.isKnownTestsEnabled ?? false @@ -336,14 +340,22 @@ class JestPlugin extends CiPlugin { this.activeTestSpan = span }) - this.addSub('ci:jest:test:finish', ({ status, testStartLine, isQuarantined }) => { + this.addSub('ci:jest:test:finish', ({ + status, + testStartLine, + attemptToFixPassed, + failedAllTests + }) => { const span = storage('legacy').getStore().span span.setTag(TEST_STATUS, status) if (testStartLine) { span.setTag(TEST_SOURCE_START, testStartLine) } - if (isQuarantined) { - span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + if (failedAllTests) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') } const spanTags = span.context()._tags @@ -407,7 +419,11 @@ class JestPlugin extends CiPlugin { testSourceFile, isNew, isEfdRetry, - isJestRetry + isAttemptToFix, + isAttemptToFixRetry, + isJestRetry, + isDisabled, + isQuarantined } = test const extraTags = { @@ -425,6 +441,23 @@ class JestPlugin extends CiPlugin { extraTags[JEST_DISPLAY_NAME] = displayName } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + } + + if (isAttemptToFixRetry) { + extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'attempt_to_fix' + } + + if (isDisabled) { + extraTags[TEST_MANAGEMENT_IS_DISABLED] = 'true' + } + + if (isQuarantined) { + extraTags[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' + } + if (isNew) { extraTags[TEST_IS_NEW] = 'true' if (isEfdRetry) { diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 9df19700292..ef3867804a9 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -17,7 +17,6 @@ const { TEST_CODE_OWNERS, ITR_CORRELATION_ID, TEST_SOURCE_FILE, - removeEfdStringFromTestName, TEST_IS_NEW, TEST_IS_RETRY, TEST_EARLY_FLAKE_ENABLED, @@ -34,7 +33,10 @@ const { TEST_RETRY_REASON, TEST_MANAGEMENT_ENABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -199,7 +201,14 @@ class MochaPlugin extends CiPlugin { this.tracer._exporter.flush() }) - this.addSub('ci:mocha:test:finish', ({ status, hasBeenRetried, isLastRetry }) => { + this.addSub('ci:mocha:test:finish', ({ + status, + hasBeenRetried, + isLastRetry, + hasFailedAllRetries, + attemptToFixPassed, + isAttemptToFixRetry + }) => { const store = storage('legacy').getStore() const span = store?.span @@ -208,6 +217,16 @@ class MochaPlugin extends CiPlugin { if (hasBeenRetried) { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + if (isAttemptToFixRetry) { + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + } const spanTags = span.context()._tags this.telemetry.ciVisEvent( @@ -403,18 +422,18 @@ class MochaPlugin extends CiPlugin { startTestSpan (testInfo) { const { + testName, testSuiteAbsolutePath, title, isNew, isEfdRetry, testStartLine, isParallel, + isAttemptToFix, isDisabled, isQuarantined } = testInfo - const testName = removeEfdStringFromTestName(testInfo.testName) - const extraTags = {} const testParametersString = getTestParametersString(this._testTitleToParams, title) if (testParametersString) { @@ -429,6 +448,10 @@ class MochaPlugin extends CiPlugin { extraTags[MOCHA_IS_PARALLEL] = 'true' } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + } + if (isDisabled) { extraTags[TEST_MANAGEMENT_IS_DISABLED] = 'true' } diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 9c75c690bc4..bc073a4f9d5 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -20,7 +20,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_ENABLED, TEST_BROWSER_NAME, - TEST_MANAGEMENT_IS_DISABLED + TEST_MANAGEMENT_IS_DISABLED, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_HAS_FAILED_ALL_RETRIES, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT } = require('../../dd-trace/src/constants') @@ -162,7 +165,11 @@ class PlaywrightPlugin extends CiPlugin { isNew, isEfdRetry, isRetry, - isQuarantined + isAttemptToFix, + isQuarantined, + isAttemptToFixRetry, + hasFailedAllRetries, + hasPassedAttemptToFixRetries }) => { const store = storage('legacy').getStore() const span = store && store.span @@ -186,6 +193,19 @@ class PlaywrightPlugin extends CiPlugin { if (isRetry) { span.setTag(TEST_IS_RETRY, 'true') } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } + if (isAttemptToFix) { + span.setTag(TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true') + } + if (isAttemptToFixRetry) { + span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'attempt_to_fix') + } + if (hasPassedAttemptToFixRetries) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } if (isQuarantined) { span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') } diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 06a5257d905..193ec152dc0 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -23,7 +23,10 @@ const { TEST_MANAGEMENT_IS_QUARANTINED, TEST_MANAGEMENT_IS_DISABLED, DD_CAPABILITIES_EARLY_FLAKE_DETECTION, - DD_CAPABILITIES_AUTO_TEST_RETRIES + DD_CAPABILITIES_AUTO_TEST_RETRIES, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, + TEST_HAS_FAILED_ALL_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -53,18 +56,30 @@ class VitestPlugin extends CiPlugin { onDone(!testsForThisTestSuite.includes(testName)) }) + this.addSub('ci:vitest:test:is-attempt-to-fix', ({ + testManagementTests, + testSuiteAbsolutePath, + testName, + onDone + }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const { isAttemptToFix } = this.getTestProperties(testManagementTests, testSuite, testName) + + onDone(isAttemptToFix ?? false) + }) + this.addSub('ci:vitest:test:is-disabled', ({ testManagementTests, testSuiteAbsolutePath, testName, onDone }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const { isDisabled } = this.getTestProperties(testManagementTests, testSuite, testName) - onDone(isDisabled ?? false) + onDone(isDisabled) }) this.addSub('ci:vitest:test:is-quarantined', ({ testManagementTests, testSuiteAbsolutePath, testName, onDone }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const { isQuarantined } = this.getTestProperties(testManagementTests, testSuite, testName) - onDone(isQuarantined ?? false) + onDone(isQuarantined) }) this.addSub('ci:vitest:is-early-flake-detection-faulty', ({ @@ -85,9 +100,12 @@ class VitestPlugin extends CiPlugin { testSuiteAbsolutePath, isRetry, isNew, + isAttemptToFix, isQuarantined, + isDisabled, mightHitProbe, - isRetryReasonEfd + isRetryReasonEfd, + isRetryReasonAttemptToFix }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const store = storage('legacy').getStore() @@ -104,9 +122,18 @@ class VitestPlugin extends CiPlugin { if (isRetryReasonEfd) { extraTags[TEST_RETRY_REASON] = 'efd' } + if (isAttemptToFix) { + extraTags[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] = 'true' + } + if (isRetryReasonAttemptToFix) { + extraTags[TEST_RETRY_REASON] = 'attempt_to_fix' + } if (isQuarantined) { extraTags[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' } + if (isDisabled) { + extraTags[TEST_MANAGEMENT_IS_DISABLED] = 'true' + } const span = this.startTestSpan( testName, @@ -124,7 +151,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + this.addSub('ci:vitest:test:finish-time', ({ status, task, attemptToFixPassed }) => { const store = storage('legacy').getStore() const span = store?.span @@ -132,6 +159,11 @@ class VitestPlugin extends CiPlugin { // this is because the test might fail at a `afterEach` hook if (span) { span.setTag(TEST_STATUS, status) + + if (attemptToFixPassed) { + span.setTag(TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED, 'true') + } + this.taskToFinishTime.set(task, span._getTime()) } }) @@ -150,7 +182,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:error', ({ duration, error, shouldSetProbe, promises }) => { + this.addSub('ci:vitest:test:error', ({ duration, error, shouldSetProbe, promises, hasFailedAllRetries }) => { const store = storage('legacy').getStore() const span = store?.span @@ -172,6 +204,9 @@ class VitestPlugin extends CiPlugin { if (error) { span.setTag('error', error) } + if (hasFailedAllRetries) { + span.setTag(TEST_HAS_FAILED_ALL_RETRIES, 'true') + } if (duration) { span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds } else { @@ -327,10 +362,10 @@ class VitestPlugin extends CiPlugin { } getTestProperties (testManagementTests, testSuite, testName) { - const { disabled: isDisabled, quarantined: isQuarantined } = + const { attempt_to_fix: isAttemptToFix, disabled: isDisabled, quarantined: isQuarantined } = testManagementTests?.vitest?.suites?.[testSuite]?.tests?.[testName]?.properties || {} - return { isDisabled, isQuarantined } + return { isAttemptToFix, isDisabled, isQuarantined } } } diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index a87078ed7e7..dbbcd2f0f0b 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -215,7 +215,8 @@ class CiVisibilityExporter extends AgentInfoExporter { isFlakyTestRetriesEnabled, isDiEnabled, isKnownTestsEnabled, - isTestManagementEnabled + isTestManagementEnabled, + testManagementAttemptToFixRetries } = remoteConfiguration return { isCodeCoverageEnabled, @@ -229,7 +230,9 @@ class CiVisibilityExporter extends AgentInfoExporter { flakyTestRetriesCount: this._config.flakyTestRetriesCount, isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled, isKnownTestsEnabled, - isTestManagementEnabled: isTestManagementEnabled && this._config.isTestManagementEnabled + isTestManagementEnabled: isTestManagementEnabled && this._config.isTestManagementEnabled, + testManagementAttemptToFixRetries: + testManagementAttemptToFixRetries ?? this._config.testManagementAttemptToFixRetries } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index 39d9fd1e11b..ebd00ea7574 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -113,7 +113,9 @@ function getLibraryConfiguration ({ isFlakyTestRetriesEnabled, isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled, isKnownTestsEnabled, - isTestManagementEnabled: (testManagementConfig?.enabled ?? false) + isTestManagementEnabled: (testManagementConfig?.enabled ?? false), + testManagementAttemptToFixRetries: + testManagementConfig?.attempt_to_fix_retries } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js b/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js index 5aca25e9e19..f5e897f0782 100644 --- a/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js +++ b/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js @@ -6,7 +6,8 @@ function getTestManagementTests ({ isEvpProxy, evpProxyPrefix, isGzipCompatible, - repositoryUrl + repositoryUrl, + commitMessage }, done) { const options = { path: '/api/v2/test/libraries/test-management/tests', @@ -39,7 +40,8 @@ function getTestManagementTests ({ id: id().toString(10), type: 'ci_app_libraries_tests_request', attributes: { - repository_url: repositoryUrl + repository_url: repositoryUrl, + commit_message: commitMessage } } }) diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index e1940265706..63c716acb32 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -41,7 +41,14 @@ const { TELEMETRY_EVENT_CREATED, TELEMETRY_ITR_SKIPPED } = require('../ci-visibility/telemetry') -const { CI_PROVIDER_NAME, GIT_REPOSITORY_URL, GIT_COMMIT_SHA, GIT_BRANCH, CI_WORKSPACE_PATH } = require('./util/tags') +const { + CI_PROVIDER_NAME, + GIT_REPOSITORY_URL, + GIT_COMMIT_SHA, + GIT_BRANCH, + CI_WORKSPACE_PATH, + GIT_COMMIT_MESSAGE +} = require('./util/tags') const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env') const getDiClient = require('../ci-visibility/dynamic-instrumentation') @@ -251,7 +258,8 @@ module.exports = class CiPlugin extends Plugin { [RUNTIME_VERSION]: runtimeVersion, [GIT_BRANCH]: branch, [CI_PROVIDER_NAME]: ciProviderName, - [CI_WORKSPACE_PATH]: repositoryRoot + [CI_WORKSPACE_PATH]: repositoryRoot, + [GIT_COMMIT_MESSAGE]: commitMessage } = this.testEnvironmentMetadata this.repositoryRoot = repositoryRoot || process.cwd() @@ -269,7 +277,8 @@ module.exports = class CiPlugin extends Plugin { runtimeName, runtimeVersion, branch, - testLevel: 'suite' + testLevel: 'suite', + commitMessage } } diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 92d62424d48..c6e95d5a45b 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -58,6 +58,7 @@ const TEST_IS_RETRY = 'test.is_retry' const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled' const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason' const TEST_RETRY_REASON = 'test.retry_reason' +const TEST_HAS_FAILED_ALL_RETRIES = 'test.has_failed_all_retries' const CI_APP_ORIGIN = 'ciapp-test' @@ -120,9 +121,16 @@ const DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX = 'snapshot_id' const DI_DEBUG_ERROR_FILE_SUFFIX = 'file' const DI_DEBUG_ERROR_LINE_SUFFIX = 'line' +// Test Management tags +const TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX = 'test.test_management.is_attempt_to_fix' const TEST_MANAGEMENT_IS_DISABLED = 'test.test_management.is_test_disabled' const TEST_MANAGEMENT_IS_QUARANTINED = 'test.test_management.is_quarantined' const TEST_MANAGEMENT_ENABLED = 'test.test_management.enabled' +const TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED = 'test.test_management.attempt_to_fix_passed' + +// Test Management utils strings +const ATTEMPT_TO_FIX_STRING = "Retried by Datadog's Test Management" +const ATTEMPT_TEST_NAME_REGEX = new RegExp(ATTEMPT_TO_FIX_STRING + ' \\(#\\d+\\): ', 'g') module.exports = { TEST_CODE_OWNERS, @@ -155,6 +163,7 @@ module.exports = { TEST_EARLY_FLAKE_ENABLED, TEST_EARLY_FLAKE_ABORT_REASON, TEST_RETRY_REASON, + TEST_HAS_FAILED_ALL_RETRIES, getTestEnvironmentMetadata, getTestParametersString, finishAllTraceSpans, @@ -192,7 +201,9 @@ module.exports = { EFD_STRING, EFD_TEST_NAME_REGEX, removeEfdStringFromTestName, + removeAttemptToFixStringFromTestName, addEfdStringToTestName, + addAttemptToFixStringToTestName, getIsFaultyEarlyFlakeDetection, TEST_BROWSER_DRIVER, TEST_BROWSER_DRIVER_VERSION, @@ -212,9 +223,11 @@ module.exports = { DI_DEBUG_ERROR_LINE_SUFFIX, getFormattedError, DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, TEST_MANAGEMENT_IS_DISABLED, TEST_MANAGEMENT_IS_QUARANTINED, - TEST_MANAGEMENT_ENABLED + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED } // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19 @@ -622,10 +635,18 @@ function addEfdStringToTestName (testName, numAttempt) { return `${EFD_STRING} (#${numAttempt}): ${testName}` } +function addAttemptToFixStringToTestName (testName, numAttempt) { + return `${ATTEMPT_TO_FIX_STRING} (#${numAttempt}): ${testName}` +} + function removeEfdStringFromTestName (testName) { return testName.replace(EFD_TEST_NAME_REGEX, '') } +function removeAttemptToFixStringFromTestName (testName) { + return testName.replace(ATTEMPT_TEST_NAME_REGEX, '') +} + function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faultyThresholdPercentage) { let newSuites = 0 for (const suite of projectSuites) {