From 265b8271c450902c39f43e063487d000da2d798b Mon Sep 17 00:00:00 2001 From: Muaz Othman Date: Tue, 3 Oct 2023 15:25:28 -0500 Subject: [PATCH] Initial work to add burn-in to test execution logic --- packages/driver/src/cypress/mocha.ts | 149 +++++++++++++++++++++------ 1 file changed, 117 insertions(+), 32 deletions(-) diff --git a/packages/driver/src/cypress/mocha.ts b/packages/driver/src/cypress/mocha.ts index e586dc514391..6283580eac6a 100644 --- a/packages/driver/src/cypress/mocha.ts +++ b/packages/driver/src/cypress/mocha.ts @@ -37,10 +37,29 @@ delete (window as any).Mocha export const SKIPPED_DUE_TO_BROWSER_MESSAGE = ' (skipped due to browser)' +type LatestScore = null | -2 | -1 | 0 | 1 + +type AttemptStrategy = 'RETRY' | 'BURN_IN' | 'NONE' + +type ReasonToStop = +| 'PASSED_FIRST_ATTEMPT' // no burn-in needed +| 'PASSED_BURN_IN' // achieved burn-in +| 'PASSED_MET_THRESHOLD' // passed after reaching threshold for strategy 'detect-flake-and-pass-on-threshold' +| 'FAILED_NO_RETRIES' // failed and no retries +| 'FAILED_REACHED_MAX_RETRIES' // failed after reaching max retries +| 'FAILED_DID_NOT_MEET_THRESHOLD' // failed since it's impossible to meet threshold for strategy 'detect-flake-and-pass-on-threshold' +| 'FAILED_STOPPED_ON_FLAKE' // failed with one attempt passing and using strategy 'detect-flake-but-always-fail' with `stopIfAnyPassed` set to true +// NOTE: can we detect this? how? the goal is to avoid retrying a test that failed because of a hook failure +| 'FAILED_HOOK_FAILED' // failed because a hook failed + interface CypressTest extends Mocha.Test { prevAttempts: CypressTest[] final?: boolean forceState?: 'passed' + latestScore?: LatestScore + thisAttemptInitialStrategy?: AttemptStrategy + nextAttemptStrategy?: AttemptStrategy + reasonToStop?: ReasonToStop } type Strategy = 'detect-flake-and-pass-on-threshold' | 'detect-flake-but-always-fail' | undefined @@ -56,8 +75,33 @@ type Options = T extends 'detect-flake-and-pass-on-threshold' ? } : undefined +type CompleteBurnInConfig = { + enabled: boolean + default: number + flaky: number +} + +function getNeededBurnInAttempts (latestScore: LatestScore, burnInConfig: CompleteBurnInConfig) { + if (burnInConfig.enabled === false) { + return 0 + } + + switch (latestScore) { + case null: return burnInConfig.default // this means the cloud determined the test is new or modified + case 0: return burnInConfig.default // this means the cloud determined the test was failing with no flake + case -1: return burnInConfig.flaky // this means the cloud determined the test was flaky + case -2: return 0 // this means the cloud couldn't determine the score + case 1: return 0 // this means the cloud determined the test graduated burn-in + default: return 0 + } +} + // NOTE: 'calculateTestStatus' is marked as an individual function to make functionality easier to test. -export function calculateTestStatus (test: CypressTest, strategy: Strategy, options: Options) { +export function calculateTestStatus (test: CypressTest, strategy: Strategy, options: Options, completeBurnInConfig: CompleteBurnInConfig) { + const latestScore = test.latestScore ?? null + + const neededBurnInAttempts = getNeededBurnInAttempts(latestScore, completeBurnInConfig) + // @ts-expect-error const totalAttemptsAlreadyExecuted = test.currentRetry() + 1 let shouldAttemptsContinue: boolean = true @@ -73,10 +117,17 @@ export function calculateTestStatus (test: CypressTest, strategy: Strategy, opti failedTests.push(test) } + if (!test.prevAttempts?.length) { + test.thisAttemptInitialStrategy = 'NONE' + } else { + test.thisAttemptInitialStrategy = test.prevAttempts[test.prevAttempts.length - 1].nextAttemptStrategy + } + // If there is AT LEAST one failed test attempt, we know we need to apply retry logic. // Otherwise, the test might be burning in (not implemented yet) OR the test passed on the first attempt, // meaning retry logic does NOT need to be applied. if (failedTests.length > 0) { + test.nextAttemptStrategy = 'RETRY' const maxAttempts = test.retries() + 1 const remainingAttempts = maxAttempts - totalAttemptsAlreadyExecuted const passingAttempts = passedTests.length @@ -95,40 +146,68 @@ export function calculateTestStatus (test: CypressTest, strategy: Strategy, opti ((options as Options<'detect-flake-but-always-fail'>).stopIfAnyPassed || false) : null - // Do we have the required amount of passes? If yes, we no longer need to keep running the test. - if (strategy !== 'detect-flake-but-always-fail' && passingAttempts >= (passesRequired as number)) { - outerTestStatus = 'passed' - test.final = true - shouldAttemptsContinue = false - } else if (totalAttemptsAlreadyExecuted < maxAttempts && - ( - // For strategy "detect-flake-and-pass-on-threshold" or no strategy (current GA retries): - // If we haven't met our max attempt limit AND we have enough remaining attempts that can satisfy the passing requirement. - // retry the test. - (strategy !== 'detect-flake-but-always-fail' && remainingAttempts >= (neededPassingAttemptsLeft as number)) || - // For strategy "detect-flake-but-always-fail": - // If we haven't met our max attempt limit AND - // stopIfAnyPassed is false OR - // stopIfAnyPassed is true and no tests have passed yet. - // retry the test. - (strategy === 'detect-flake-but-always-fail' && (!stopIfAnyPassed || stopIfAnyPassed && passingAttempts === 0)) - )) { - test.final = false + switch (strategy) { + case 'detect-flake-and-pass-on-threshold': + if (passingAttempts >= (passesRequired as number)) { + // we met the threshold, so we can stop retrying and pass the test + outerTestStatus = 'passed' + test.final = true + shouldAttemptsContinue = false + test.reasonToStop = 'PASSED_MET_THRESHOLD' + } else if (remainingAttempts < (neededPassingAttemptsLeft as number)) { + // we don't have enough remaining attempts to meet the threshold, so we should stop retrying and fail the test + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_DID_NOT_MEET_THRESHOLD' + } else { + // we haven't met the threshold, but we have enough remaining attempts to meet the threshold, so we should retry the test + test.final = false + shouldAttemptsContinue = true + } + + break + case 'detect-flake-but-always-fail': + if (stopIfAnyPassed && passingAttempts > 0) { + // we have a passing attempt and we should stop retrying and fail the test + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_STOPPED_ON_FLAKE' + } else if (remainingAttempts === 0) { + // we have no remaining attempts and we should stop retrying and fail the test + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_REACHED_MAX_RETRIES' + } else { + // we have remaining attempts and we should retry the test + test.final = false + shouldAttemptsContinue = true + } + + break + default: + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_NO_RETRIES' + } + } else { + test.nextAttemptStrategy = 'BURN_IN' + if (neededBurnInAttempts > passedTests.length) { shouldAttemptsContinue = true + test.final = false } else { - // Otherwise, we should stop retrying the test. - outerTestStatus = 'failed' - test.final = true - // If an outerStatus is 'failed', but the last test attempt was 'passed', we need to force the status so mocha doesn't flag the test attempt as failed. - // This is a common use case with 'detect-flake-but-always-fail', where we want to display the last attempt as 'passed' but fail the test. - test.forceState = test.state === 'passed' ? test.state : undefined + test.reasonToStop = neededBurnInAttempts > 0 ? 'PASSED_BURN_IN' : 'PASSED_FIRST_ATTEMPT' + outerTestStatus = 'passed' shouldAttemptsContinue = false + test.final = true } - } else { - // retry logic did not need to be applied and the test passed. - outerTestStatus = 'passed' - shouldAttemptsContinue = false - test.final = true } return { @@ -445,9 +524,15 @@ function createCalculateTestStatus (Cypress: Cypress.Cypress) { // 'calculateTestStatus', which makes the function easier to unit-test Test.prototype.calculateTestStatus = function () { let retriesConfig = Cypress.config('retries') + let burnInConfig = Cypress.config('experimentalBurnIn') + + // TODO: inject test.latestScore value coming from the cloud via actions + + // TODO: use the config value coming from the cloud via actions https://cypress-io.atlassian.net/browse/CYCLOUD-1140 + const completeBurnInConfig = typeof burnInConfig === 'boolean' ? { enabled: burnInConfig, default: 3, flaky: 5 } : { enabled: true, ...burnInConfig } // @ts-expect-error - return calculateTestStatus(this, retriesConfig?.experimentalStrategy, retriesConfig?.experimentalOptions) + return calculateTestStatus(this, retriesConfig?.experimentalStrategy, retriesConfig?.experimentalOptions, completeBurnInConfig) } }