From e51739c4a9d9e11c3fb701f4285c31a183297205 Mon Sep 17 00:00:00 2001 From: "artem.sukhinin" Date: Fri, 15 May 2026 17:48:10 +0300 Subject: [PATCH 1/2] WAT-5697 --- core/cli-config/src/default-config.ts | 1 + core/cli-config/test/arguments-parser.spec.ts | 9 + core/cli-config/test/get-config.spec.ts | 13 + core/cli/src/index.ts | 6 + .../src/test-run-controller.ts | 69 +++- .../test/test-run-controller.spec.ts | 304 ++++++++++++++++++ core/types/src/config.ts | 1 + docs/config.md | 21 ++ 8 files changed, 420 insertions(+), 4 deletions(-) diff --git a/core/cli-config/src/default-config.ts b/core/cli-config/src/default-config.ts index eec06c75f..938851200 100644 --- a/core/cli-config/src/default-config.ts +++ b/core/cli-config/src/default-config.ts @@ -14,6 +14,7 @@ export const defaultConfiguration: IConfig = { maxWriteThreadCount: 2, plugins: [], retryCount: 3, + forceRetryCount: 0, retryDelay: 2000, testTimeout: 15 * 60 * 1000, logLevel: LogLevel.info, diff --git a/core/cli-config/test/arguments-parser.spec.ts b/core/cli-config/test/arguments-parser.spec.ts index 8ad3df014..3eac03da2 100644 --- a/core/cli-config/test/arguments-parser.spec.ts +++ b/core/cli-config/test/arguments-parser.spec.ts @@ -22,6 +22,8 @@ describe('argument parser', () => { const pluginsSet = ['plugin1', 'plugin2', 'plugin3']; const customFieldSet = '#P0,#P1,#P2'; const customField = '#P0'; + const forceRetryCount = 3; + const rcForceRetryCount = 4; const argv = [ '', @@ -32,6 +34,7 @@ describe('argument parser', () => { `--plugins=${pluginsSet[0]}`, `--plugins=${pluginsSet[1]}`, `--plugins=${pluginsSet[2]}`, + `--force-retry-count=${forceRetryCount}`, // value without assign '--tests', customTestsPath, @@ -41,6 +44,8 @@ describe('argument parser', () => { customFieldSet, '--my-namespaced.second-custom-field', customField, + '--rc.force-retry-count', + `${rcForceRetryCount}`, ]; const args = getArguments(argv); @@ -49,6 +54,7 @@ describe('argument parser', () => { config: customConfigPath, tests: customTestsPath, plugins: pluginsSet, + forceRetryCount, customField: customFieldSet, /* are the following needed ??? - looks like undocumented feature for early version // myNamespacedCustomField: customFieldSet, @@ -58,6 +64,9 @@ describe('argument parser', () => { customField: customFieldSet, secondCustomField: customField, }, + rc: { + forceRetryCount: rcForceRetryCount, + }, }; chai.expect(args).to.be.deep.equal(expected); diff --git a/core/cli-config/test/get-config.spec.ts b/core/cli-config/test/get-config.spec.ts index ccc50b503..0fb5fe254 100644 --- a/core/cli-config/test/get-config.spec.ts +++ b/core/cli-config/test/get-config.spec.ts @@ -63,6 +63,19 @@ describe('Get config', () => { chai.expect(config).to.have.property('workerLimit', override); }); + it('should override force retry count with arguments', async () => { + const forceRetryCount = 7; + + const config = await getConfig([ + `--force-retry-count=${forceRetryCount}`, + ]); + + chai.expect(config).to.have.property( + 'forceRetryCount', + forceRetryCount, + ); + }); + it('should override every resolved config fields with arguments', async () => { const override = 'argumentsConfig'; diff --git a/core/cli/src/index.ts b/core/cli/src/index.ts index 107f134ae..abdbae108 100644 --- a/core/cli/src/index.ts +++ b/core/cli/src/index.ts @@ -35,6 +35,12 @@ createField('retryCount', { type: 'number', }); +createField('forceRetryCount', { + describe: + 'Total forced attempts for every test; 0 disables force retry mode', + type: 'number', +}); + createField('retryDelay', { describe: 'Time of delay before retry', type: 'number', diff --git a/core/test-run-controller/src/test-run-controller.ts b/core/test-run-controller/src/test-run-controller.ts index b3e5b1ca2..00c9f24b2 100644 --- a/core/test-run-controller/src/test-run-controller.ts +++ b/core/test-run-controller/src/test-run-controller.ts @@ -227,6 +227,40 @@ export class TestRunController }; } + private getForceRetryCount(): number { + const forceRetryCount = Number(this.config.forceRetryCount); + + if (!Number.isInteger(forceRetryCount) || forceRetryCount <= 0) { + return 0; + } + + return forceRetryCount; + } + + private isForceRetryMode(): boolean { + return this.getForceRetryCount() > 0; + } + + private shouldScheduleNextForcedAttempt(queueItem: IQueuedTest): boolean { + return queueItem.retryCount < this.getForceRetryCount() - 1; + } + + private getNextRetryQueueItem(queueItem: IQueuedTest): IQueuedTest { + return this.getQueueItemWithRunData({ + ...queueItem, + retryCount: queueItem.retryCount + 1, + }); + } + + private scheduleNextForcedAttempt( + queueItem: IQueuedTest, + queue: TestQueue, + ): void { + if (this.shouldScheduleNextForcedAttempt(queueItem)) { + queue.push(this.getNextRetryQueueItem(queueItem)); + } + } + private async prepareTests(testFiles: Array): Promise { const testQueue = new Array(testFiles.length); @@ -265,6 +299,32 @@ export class TestRunController throw error; } + if (this.isForceRetryMode()) { + this.errors.push(error); + + if (this.shouldScheduleNextForcedAttempt(queueItem)) { + await delay(this.config.retryDelay || 0); + + await this.callHook( + TestRunControllerPlugins.beforeTestRetry, + queueItem, + error, + this.getWorkerMeta(worker), + ); + + queue.push(this.getNextRetryQueueItem(queueItem)); + } else { + await this.callHook( + TestRunControllerPlugins.afterTest, + queueItem, + error, + this.getWorkerMeta(worker), + ); + } + + return; + } + const shouldNotRetry = await this.callHook( TestRunControllerPlugins.shouldNotRetry, false, @@ -278,10 +338,7 @@ export class TestRunController ) { await delay(this.config.retryDelay || 0); - const copyQueueItem = this.getQueueItemWithRunData({ - ...queueItem, - retryCount: queueItem.retryCount + 1, - }); + const copyQueueItem = this.getNextRetryQueueItem(queueItem); queue.push(copyQueueItem); @@ -369,6 +426,10 @@ export class TestRunController null, this.getWorkerMeta(worker), ); + + if (this.isForceRetryMode()) { + this.scheduleNextForcedAttempt(queuedTest, queue); + } } catch (error) { if (isRejectedByTimeout) { await worker.kill('SIGABRT'); diff --git a/core/test-run-controller/test/test-run-controller.spec.ts b/core/test-run-controller/test/test-run-controller.spec.ts index d1d92868b..f5e5791d8 100644 --- a/core/test-run-controller/test/test-run-controller.spec.ts +++ b/core/test-run-controller/test/test-run-controller.spec.ts @@ -3,6 +3,7 @@ import * as chai from 'chai'; import {TestWorkerMock} from '@testring/test-utils'; +import {IFile, ITestWorker, ITestWorkerInstance} from '@testring/types'; import {TestRunControllerPlugins} from '@testring/types/src/test-run-controller'; import {TestRunController} from '../src/test-run-controller'; @@ -17,6 +18,157 @@ const generateTestFile = (index: number) => ({ const generateTestFiles = (count: number) => Array.from({length: count}, (_v, i) => generateTestFile(i)); +const testDelay = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +type ShouldFailAttempt = (path: string, attempt: number) => boolean; + +interface IRecordingTestWorkerOptions { + executionDelay?: number; + shouldFailAttempt?: ShouldFailAttempt; +} + +class RecordingTestWorkerState { + public killCalls = 0; + + public maxTotalActive = 0; + + public readonly attemptsByPath = new Map(); + + public readonly retryFlagsByPath = new Map(); + + public readonly maxActiveByPath = new Map(); + + public readonly errors: Error[] = []; + + private totalActive = 0; + + private readonly activeByPath = new Map(); + + private readonly options: IRecordingTestWorkerOptions; + + constructor(options: IRecordingTestWorkerOptions = {}) { + this.options = options; + } + + public async execute(file: IFile, parameters: any): Promise { + const attempt = this.attemptsByPath.get(file.path) || 0; + this.attemptsByPath.set(file.path, attempt + 1); + this.storeRetryFlag(file.path, !!parameters.runData?.isRetryRun); + this.start(file.path); + + try { + if (this.options.executionDelay) { + await testDelay(this.options.executionDelay); + } + + if (this.options.shouldFailAttempt?.(file.path, attempt)) { + const error = new Error( + `Test ${file.path} failed on attempt ${attempt}`, + ); + this.errors.push(error); + throw error; + } + } finally { + this.finish(file.path); + } + } + + private storeRetryFlag(path: string, isRetryRun: boolean): void { + const retryFlags = this.retryFlagsByPath.get(path) || []; + retryFlags.push(isRetryRun); + this.retryFlagsByPath.set(path, retryFlags); + } + + private start(path: string): void { + const pathActive = (this.activeByPath.get(path) || 0) + 1; + this.activeByPath.set(path, pathActive); + this.maxActiveByPath.set( + path, + Math.max(this.maxActiveByPath.get(path) || 0, pathActive), + ); + + this.totalActive++; + this.maxTotalActive = Math.max( + this.maxTotalActive, + this.totalActive, + ); + } + + private finish(path: string): void { + this.activeByPath.set(path, (this.activeByPath.get(path) || 1) - 1); + this.totalActive--; + } +} + +class RecordingTestWorkerInstance implements ITestWorkerInstance { + private readonly state: RecordingTestWorkerState; + + private readonly workerID: string; + + constructor( + state: RecordingTestWorkerState, + workerID: string, + ) { + this.state = state; + this.workerID = workerID; + } + + public getWorkerID(): string { + return this.workerID; + } + + public execute( + file: IFile, + parameters: any, + _envParameters: any, + ): Promise { + return this.state.execute(file, parameters); + } + + public async kill(): Promise { + this.state.killCalls++; + } +} + +class RecordingTestWorker implements ITestWorker { + public readonly state: RecordingTestWorkerState; + + private spawnedCount = 0; + + constructor(options: IRecordingTestWorkerOptions = {}) { + this.state = new RecordingTestWorkerState(options); + } + + public spawn(): ITestWorkerInstance { + this.spawnedCount++; + + return new RecordingTestWorkerInstance( + this.state, + `worker/${this.spawnedCount}`, + ); + } + + public getAttemptCount(path: string): number { + return this.state.attemptsByPath.get(path) || 0; + } + + public getRetryFlags(path: string): boolean[] { + return this.state.retryFlagsByPath.get(path) || []; + } + + public getMaxActiveByPath(path: string): number { + return this.state.maxActiveByPath.get(path) || 0; + } + + public getTotalAttemptCount(): number { + return [...this.state.attemptsByPath.values()].reduce( + (sum, count) => sum + count, + 0, + ); + } +} + describe('TestRunController', () => { it('should fail if zero workers are passed', async () => { const workerLimit = 0; @@ -269,6 +421,158 @@ describe('TestRunController', () => { ); }); + it('should force every passing test to run requested count of attempts', async () => { + const forceRetryCount = 3; + const config = { + workerLimit: 2, + retryDelay: 0, + retryCount: 0, + forceRetryCount, + testTimeout: DEFAULT_TIMEOUT, + } as any; + + const tests = generateTestFiles(2); + const testWorkerMock = new RecordingTestWorker(); + const testRunController = new TestRunController(config, testWorkerMock); + + const errors = await testRunController.runQueue(tests); + + chai.expect(errors).to.be.equal(null); + tests.forEach((test) => { + chai.expect(testWorkerMock.getAttemptCount(test.path)).to.equal( + forceRetryCount, + ); + chai.expect(testWorkerMock.getRetryFlags(test.path)).to.deep.equal([ + false, + true, + true, + ]); + }); + }); + + it('should force every failing test to run requested count of attempts and return all failures', async () => { + const forceRetryCount = 3; + const config = { + workerLimit: 2, + retryDelay: 0, + retryCount: 0, + forceRetryCount, + testTimeout: DEFAULT_TIMEOUT, + } as any; + + const tests = generateTestFiles(2); + const testWorkerMock = new RecordingTestWorker({ + shouldFailAttempt: () => true, + }); + const testRunController = new TestRunController(config, testWorkerMock); + + const errors = (await testRunController.runQueue(tests)) as Error[]; + + chai.expect(errors).to.be.lengthOf(tests.length * forceRetryCount); + chai.expect(errors).to.have.members(testWorkerMock.state.errors); + tests.forEach((test) => { + chai.expect(testWorkerMock.getAttemptCount(test.path)).to.equal( + forceRetryCount, + ); + }); + }); + + it('should not overlap forced attempts for the same test', async () => { + const forceRetryCount = 3; + const config = { + workerLimit: 3, + retryDelay: 0, + retryCount: 0, + forceRetryCount, + testTimeout: DEFAULT_TIMEOUT, + } as any; + + const tests = generateTestFiles(2); + const testWorkerMock = new RecordingTestWorker({ + executionDelay: 20, + }); + const testRunController = new TestRunController(config, testWorkerMock); + + await testRunController.runQueue(tests); + + tests.forEach((test) => { + chai.expect(testWorkerMock.getMaxActiveByPath(test.path)).to.equal( + 1, + ); + chai.expect(testWorkerMock.getAttemptCount(test.path)).to.equal( + forceRetryCount, + ); + }); + }); + + it('should allow different forced tests to overlap up to worker limit', async () => { + const config = { + workerLimit: 2, + retryDelay: 0, + retryCount: 0, + forceRetryCount: 2, + testTimeout: DEFAULT_TIMEOUT, + } as any; + + const tests = generateTestFiles(2); + const testWorkerMock = new RecordingTestWorker({ + executionDelay: 20, + }); + const testRunController = new TestRunController(config, testWorkerMock); + + await testRunController.runQueue(tests); + + chai.expect(testWorkerMock.state.maxTotalActive).to.equal(2); + }); + + it('should ignore configured retry count in force retry mode', async () => { + const forceRetryCount = 2; + const config = { + workerLimit: 1, + retryDelay: 0, + retryCount: 5, + forceRetryCount, + testTimeout: DEFAULT_TIMEOUT, + } as any; + + const tests = generateTestFiles(1); + const testWorkerMock = new RecordingTestWorker({ + shouldFailAttempt: () => true, + }); + const testRunController = new TestRunController(config, testWorkerMock); + + const errors = (await testRunController.runQueue(tests)) as Error[]; + + chai.expect(testWorkerMock.getTotalAttemptCount()).to.equal( + forceRetryCount, + ); + chai.expect(errors).to.be.lengthOf(forceRetryCount); + }); + + it('should preserve existing retry behavior when force retry count is zero', async () => { + const retryCount = 2; + const config = { + workerLimit: 1, + retryDelay: 0, + retryCount, + forceRetryCount: 0, + testTimeout: DEFAULT_TIMEOUT, + } as any; + + const tests = generateTestFiles(1); + const testWorkerMock = new RecordingTestWorker({ + shouldFailAttempt: () => true, + }); + const testRunController = new TestRunController(config, testWorkerMock); + + const errors = (await testRunController.runQueue(tests)) as Error[]; + + chai.expect(testWorkerMock.getTotalAttemptCount()).to.equal( + retryCount + 1, + ); + chai.expect(errors).to.be.lengthOf(1); + }); + it('should not use retries when test fails', async () => { const testsCount = 3; const retriesCount = 5; diff --git a/core/types/src/config.ts b/core/types/src/config.ts index e001d3610..7880a06b9 100644 --- a/core/types/src/config.ts +++ b/core/types/src/config.ts @@ -22,6 +22,7 @@ export interface IConfig extends IConfigLogger { workerLimit: number | 'local'; maxWriteThreadCount?: number; retryCount: number; + forceRetryCount?: number; retryDelay: number; testTimeout: number; tests: string; diff --git a/docs/config.md b/docs/config.md index 690533355..dadc8ce39 100644 --- a/docs/config.md +++ b/docs/config.md @@ -19,6 +19,7 @@ if it's exists, CLI arguments overrides everything. * [bail](#bail) * [workerLimit](#workerlimit) * [retryCount](#retrycount) +* [forceRetryCount](#forceretrycount) * [retryDelay](#retrydelay) * [testTimeout](#testtimeout) * [screenshots](#screenshots) @@ -176,6 +177,26 @@ $ testring run --retry-count 5
+## `forceRetryCount` + +###### `0` default + +Total forced executions per test. When greater than `0`, every queued test +runs exactly this many attempts regardless of pass/fail, and normal +failure-driven `retryCount` is ignored. + +``` +$ testring run --force-retry-count 3 +``` + +```json +{ + "forceRetryCount": 3 +} +``` + +
+ ## `retryDelay` ###### `2000` default From 72f5fda29e223571d268084dc81de9eeae9389a6 Mon Sep 17 00:00:00 2001 From: "artem.sukhinin" Date: Fri, 15 May 2026 18:21:38 +0300 Subject: [PATCH 2/2] WAT-5643 --- core/api/src/run.ts | 21 +++-- core/api/src/test-context.ts | 5 +- core/api/test/run.spec.ts | 176 +++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 12 deletions(-) create mode 100644 core/api/test/run.spec.ts diff --git a/core/api/src/run.ts b/core/api/src/run.ts index 315898c95..3ce26b6b3 100644 --- a/core/api/src/run.ts +++ b/core/api/src/run.ts @@ -21,15 +21,7 @@ export async function run(...tests: Array) { const api = new TestContext(testParameters.runData); let passed = false; - let catchedError; - - afterRun(async () => { - try { - await api.end(); - } catch (err) { - loggerClient.error(err); - } - }); + let catchedError: Error | null = null; try { await bus.startedTest(); @@ -46,7 +38,16 @@ export async function run(...tests: Array) { } catch (error) { catchedError = restructureError(error as Error); } finally { - if (passed) { + try { + await api.end(); + } catch (error) { + if (!catchedError) { + catchedError = restructureError(error as Error); + passed = false; + } + } + + if (passed && !catchedError) { loggerClient.endStep(testID, 'Test passed'); await bus.finishedTest(); diff --git a/core/api/src/test-context.ts b/core/api/src/test-context.ts index 1de1315b1..c6f38f7fd 100644 --- a/core/api/src/test-context.ts +++ b/core/api/src/test-context.ts @@ -109,8 +109,9 @@ export class TestContext { } } - return Promise.all(requests).catch((error) => { - this.logError(error); + return Promise.all(requests).catch(async (error) => { + await this.logWarning(error); + throw error; }); } diff --git a/core/api/test/run.spec.ts b/core/api/test/run.spec.ts new file mode 100644 index 000000000..431563237 --- /dev/null +++ b/core/api/test/run.spec.ts @@ -0,0 +1,176 @@ +/// + +import * as chai from 'chai'; +import sinon from 'sinon'; + +import {loggerClient} from '@testring/logger'; +import {TestEvents} from '@testring/types'; + +import {run} from '../src/run'; +import {TestContext} from '../src/test-context'; +import {testAPIController} from '../src/test-api-controller'; + +const TEST_ID = 'test.js'; +const LOG_PREFIX = '[logged inside test]'; + +type Restorable = { + restore: () => void; +}; + +type RunEvents = { + failedErrors: Error[]; + getFinishedCount: () => number; + cleanup: () => void; +}; + +function prepareTestAPI(): void { + testAPIController.setTestID(TEST_ID); + testAPIController.setTestParameters({runData: {}}); + testAPIController.setEnvironmentParameters({}); +} + +function observeRunEvents(): RunEvents { + const bus = testAPIController.getBus(); + let finishedCount = 0; + const failedErrors: Error[] = []; + + const finishedHandler = () => { + finishedCount += 1; + }; + const failedHandler = (error: Error) => { + failedErrors.push(error); + }; + + bus.on(TestEvents.finished, finishedHandler); + bus.on(TestEvents.failed, failedHandler); + + return { + failedErrors, + getFinishedCount: () => finishedCount, + cleanup: () => { + bus.removeListener(TestEvents.finished, finishedHandler); + bus.removeListener(TestEvents.failed, failedHandler); + }, + }; +} + +function track( + restorables: Restorable[], + restorable: T, +): T { + restorables.push(restorable); + + return restorable; +} + +function restoreAll(restorables: Restorable[]): void { + for (const restorable of restorables.reverse()) { + restorable.restore(); + } +} + +describe('TestContext', () => { + let restorables: Restorable[]; + + beforeEach(() => { + restorables = []; + }); + + afterEach(() => { + restoreAll(restorables); + }); + + it('should log cleanup errors as warnings and rethrow them', async () => { + const context = new TestContext({}); + const cleanupError = new Error('cleanup failed'); + const application = { + isStopped: sinon.stub().returns(false), + end: sinon.stub().rejects(cleanupError), + }; + const warn = track(restorables, sinon.stub(loggerClient, 'warn')); + + Object.defineProperty(context, 'application', { + value: application, + configurable: true, + }); + + try { + await context.end(); + chai.assert.fail('Expected context.end() to reject.'); + } catch (error) { + chai.expect(error).to.equal(cleanupError); + } + + chai.expect(warn.calledOnceWithExactly(LOG_PREFIX, cleanupError)).to.be + .equal(true); + }); +}); + +describe('run', () => { + let restorables: Restorable[]; + let events: RunEvents; + let endStep: ReturnType; + + beforeEach(() => { + restorables = []; + prepareTestAPI(); + events = observeRunEvents(); + track(restorables, sinon.stub(loggerClient, 'startStep')); + endStep = track(restorables, sinon.stub(loggerClient, 'endStep')); + }); + + afterEach(() => { + events.cleanup(); + restoreAll(restorables); + }); + + it('should finish the test when body and cleanup pass', async () => { + const end = track(restorables, sinon.stub(TestContext.prototype, 'end')); + end.resolves(); + + await run(() => undefined); + + chai.expect(end.calledOnce).to.equal(true); + chai.expect(events.getFinishedCount()).to.equal(1); + chai.expect(events.failedErrors).to.deep.equal([]); + chai.expect(endStep.calledOnceWithExactly(TEST_ID, 'Test passed')).to + .equal(true); + }); + + it('should fail the test when cleanup fails after a passed body', async () => { + const cleanupError = new Error('cleanup failed'); + const end = track(restorables, sinon.stub(TestContext.prototype, 'end')); + end.rejects(cleanupError); + + await run(() => undefined); + + chai.expect(end.calledOnce).to.equal(true); + chai.expect(events.getFinishedCount()).to.equal(0); + chai.expect(events.failedErrors).to.deep.equal([cleanupError]); + chai.expect( + endStep.calledOnceWithExactly( + TEST_ID, + 'Test failed', + cleanupError, + ), + ).to.equal(true); + }); + + it('should keep the body error primary when body and cleanup both fail', async () => { + const bodyError = new Error('body failed'); + const cleanupError = new Error('cleanup failed'); + const end = track(restorables, sinon.stub(TestContext.prototype, 'end')); + end.rejects(cleanupError); + + await run(() => { + throw bodyError; + }); + + chai.expect(end.calledOnce).to.equal(true); + chai.expect(events.getFinishedCount()).to.equal(0); + chai.expect(events.failedErrors).to.deep.equal([bodyError]); + chai.expect( + endStep.calledOnceWithExactly(TEST_ID, 'Test failed', bodyError), + ).to.equal(true); + }); +});