WAT-5697

core/api/src/run.ts

@@ -21,15 +21,7 @@ export async function run(...tests: Array<TestFunction>) {
 
     const api = new TestContext(testParameters.runData);
     let passed = false;
-    let catchedError;
-
-    afterRun(async () => {
-        try {
-            await api.end();
-        } catch (err) {
-            loggerClient.error(err);
-        }
-    });
+    let catchedError: Error | null = null;
 
     try {
         await bus.startedTest();
@@ -46,7 +38,16 @@ export async function run(...tests: Array<TestFunction>) {
     } catch (error) {
         catchedError = restructureError(error as Error);
     } finally {
-        if (passed) {
+        try {
+            await api.end();
+        } catch (error) {
+            if (!catchedError) {
+                catchedError = restructureError(error as Error);
+                passed = false;
+            }
+        }
+
+        if (passed && !catchedError) {
             loggerClient.endStep(testID, 'Test passed');
 
             await bus.finishedTest();

core/api/src/test-context.ts

@@ -109,8 +109,9 @@ export class TestContext {
             }
         }
 
-        return Promise.all(requests).catch((error) => {
-            this.logError(error);
+        return Promise.all(requests).catch(async (error) => {
+            await this.logWarning(error);
+            throw error;
         });
     }
 

core/api/test/run.spec.ts

@@ -0,0 +1,176 @@
+/// <reference types="mocha" />
+
+import * as chai from 'chai';
+import sinon from 'sinon';
+
+import {loggerClient} from '@testring/logger';
+import {TestEvents} from '@testring/types';
+
+import {run} from '../src/run';
+import {TestContext} from '../src/test-context';
+import {testAPIController} from '../src/test-api-controller';
+
+const TEST_ID = 'test.js';
+const LOG_PREFIX = '[logged inside test]';
+
+type Restorable = {
+    restore: () => void;
+};
+
+type RunEvents = {
+    failedErrors: Error[];
+    getFinishedCount: () => number;
+    cleanup: () => void;
+};
+
+function prepareTestAPI(): void {
+    testAPIController.setTestID(TEST_ID);
+    testAPIController.setTestParameters({runData: {}});
+    testAPIController.setEnvironmentParameters({});
+}
+
+function observeRunEvents(): RunEvents {
+    const bus = testAPIController.getBus();
+    let finishedCount = 0;
+    const failedErrors: Error[] = [];
+
+    const finishedHandler = () => {
+        finishedCount += 1;
+    };
+    const failedHandler = (error: Error) => {
+        failedErrors.push(error);
+    };
+
+    bus.on(TestEvents.finished, finishedHandler);
+    bus.on(TestEvents.failed, failedHandler);
+
+    return {
+        failedErrors,
+        getFinishedCount: () => finishedCount,
+        cleanup: () => {
+            bus.removeListener(TestEvents.finished, finishedHandler);
+            bus.removeListener(TestEvents.failed, failedHandler);
+        },
+    };
+}
+
+function track<T extends Restorable>(
+    restorables: Restorable[],
+    restorable: T,
+): T {
+    restorables.push(restorable);
+
+    return restorable;
+}
+
+function restoreAll(restorables: Restorable[]): void {
+    for (const restorable of restorables.reverse()) {
+        restorable.restore();
+    }
+}
+
+describe('TestContext', () => {
+    let restorables: Restorable[];
+
+    beforeEach(() => {
+        restorables = [];
+    });
+
+    afterEach(() => {
+        restoreAll(restorables);
+    });
+
+    it('should log cleanup errors as warnings and rethrow them', async () => {
+        const context = new TestContext({});
+        const cleanupError = new Error('cleanup failed');
+        const application = {
+            isStopped: sinon.stub().returns(false),
+            end: sinon.stub().rejects(cleanupError),
+        };
+        const warn = track(restorables, sinon.stub(loggerClient, 'warn'));
+
+        Object.defineProperty(context, 'application', {
+            value: application,
+            configurable: true,
+        });
+
+        try {
+            await context.end();
+            chai.assert.fail('Expected context.end() to reject.');
+        } catch (error) {
+            chai.expect(error).to.equal(cleanupError);
+        }
+
+        chai.expect(warn.calledOnceWithExactly(LOG_PREFIX, cleanupError)).to.be
+            .equal(true);
+    });
+});
+
+describe('run', () => {
+    let restorables: Restorable[];
+    let events: RunEvents;
+    let endStep: ReturnType<typeof sinon.stub>;
+
+    beforeEach(() => {
+        restorables = [];
+        prepareTestAPI();
+        events = observeRunEvents();
+        track(restorables, sinon.stub(loggerClient, 'startStep'));
+        endStep = track(restorables, sinon.stub(loggerClient, 'endStep'));
+    });
+
+    afterEach(() => {
+        events.cleanup();
+        restoreAll(restorables);
+    });
+
+    it('should finish the test when body and cleanup pass', async () => {
+        const end = track(restorables, sinon.stub(TestContext.prototype, 'end'));
+        end.resolves();
+
+        await run(() => undefined);
+
+        chai.expect(end.calledOnce).to.equal(true);
+        chai.expect(events.getFinishedCount()).to.equal(1);
+        chai.expect(events.failedErrors).to.deep.equal([]);
+        chai.expect(endStep.calledOnceWithExactly(TEST_ID, 'Test passed')).to
+            .equal(true);
+    });
+
+    it('should fail the test when cleanup fails after a passed body', async () => {
+        const cleanupError = new Error('cleanup failed');
+        const end = track(restorables, sinon.stub(TestContext.prototype, 'end'));
+        end.rejects(cleanupError);
+
+        await run(() => undefined);
+
+        chai.expect(end.calledOnce).to.equal(true);
+        chai.expect(events.getFinishedCount()).to.equal(0);
+        chai.expect(events.failedErrors).to.deep.equal([cleanupError]);
+        chai.expect(
+            endStep.calledOnceWithExactly(
+                TEST_ID,
+                'Test failed',
+                cleanupError,
+            ),
+        ).to.equal(true);
+    });
+
+    it('should keep the body error primary when body and cleanup both fail', async () => {
+        const bodyError = new Error('body failed');
+        const cleanupError = new Error('cleanup failed');
+        const end = track(restorables, sinon.stub(TestContext.prototype, 'end'));
+        end.rejects(cleanupError);
+
+        await run(() => {
+            throw bodyError;
+        });
+
+        chai.expect(end.calledOnce).to.equal(true);
+        chai.expect(events.getFinishedCount()).to.equal(0);
+        chai.expect(events.failedErrors).to.deep.equal([bodyError]);
+        chai.expect(
+            endStep.calledOnceWithExactly(TEST_ID, 'Test failed', bodyError),
+        ).to.equal(true);
+    });
+});

core/cli-config/src/default-config.ts

@@ -14,6 +14,7 @@ export const defaultConfiguration: IConfig = {
     maxWriteThreadCount: 2,
     plugins: [],
     retryCount: 3,
+    forceRetryCount: 0,
     retryDelay: 2000,
     testTimeout: 15 * 60 * 1000,
     logLevel: LogLevel.info,

core/cli-config/test/arguments-parser.spec.ts

@@ -22,6 +22,8 @@ describe('argument parser', () => {
         const pluginsSet = ['plugin1', 'plugin2', 'plugin3'];
         const customFieldSet = '#P0,#P1,#P2';
         const customField = '#P0';
+        const forceRetryCount = 3;
+        const rcForceRetryCount = 4;
 
         const argv = [
             '',
@@ -32,6 +34,7 @@ describe('argument parser', () => {
             `--plugins=${pluginsSet[0]}`,
             `--plugins=${pluginsSet[1]}`,
             `--plugins=${pluginsSet[2]}`,
+            `--force-retry-count=${forceRetryCount}`,
             // value without assign
             '--tests',
             customTestsPath,
@@ -41,6 +44,8 @@ describe('argument parser', () => {
             customFieldSet,
             '--my-namespaced.second-custom-field',
             customField,
+            '--rc.force-retry-count',
+            `${rcForceRetryCount}`,
         ];
 
         const args = getArguments(argv);
@@ -49,6 +54,7 @@ describe('argument parser', () => {
             config: customConfigPath,
             tests: customTestsPath,
             plugins: pluginsSet,
+            forceRetryCount,
             customField: customFieldSet,
             /* are the following needed ??? - looks like undocumented feature for early version
             // myNamespacedCustomField: customFieldSet,
@@ -58,6 +64,9 @@ describe('argument parser', () => {
                 customField: customFieldSet,
                 secondCustomField: customField,
             },
+            rc: {
+                forceRetryCount: rcForceRetryCount,
+            },
         };
 
         chai.expect(args).to.be.deep.equal(expected);

core/cli-config/test/get-config.spec.ts

@@ -63,6 +63,19 @@ describe('Get config', () => {
         chai.expect(config).to.have.property('workerLimit', override);
     });
 
+    it('should override force retry count with arguments', async () => {
+        const forceRetryCount = 7;
+
+        const config = await getConfig([
+            `--force-retry-count=${forceRetryCount}`,
+        ]);
+
+        chai.expect(config).to.have.property(
+            'forceRetryCount',
+            forceRetryCount,
+        );
+    });
+
     it('should override every resolved config fields with arguments', async () => {
         const override = 'argumentsConfig';
 

core/cli/src/index.ts

@@ -35,6 +35,12 @@ createField('retryCount', {
     type: 'number',
 });
 
+createField('forceRetryCount', {
+    describe:
+        'Total forced attempts for every test; 0 disables force retry mode',
+    type: 'number',
+});
+
 createField('retryDelay', {
     describe: 'Time of delay before retry',
     type: 'number',