From 64dc679545e59c6f9f71e9dabce4b413f56fd84c Mon Sep 17 00:00:00 2001 From: Appurva Murawat Date: Mon, 22 May 2023 16:28:42 +0530 Subject: [PATCH 1/6] Add support for asynchronous tests with `pending` state. --- lib/sandbox/execute.js | 7 +- lib/sandbox/pmapi-setup-runner.js | 225 ++++++++++++++++++++++++------ lib/sandbox/pmapi.js | 5 +- 3 files changed, 193 insertions(+), 44 deletions(-) diff --git a/lib/sandbox/execute.js b/lib/sandbox/execute.js index e2321ff5..66520d3d 100644 --- a/lib/sandbox/execute.js +++ b/lib/sandbox/execute.js @@ -34,7 +34,10 @@ module.exports = function (bridge, glob) { // For caching required information provided during // initialization which will be used during execution let initializationOptions = {}, - initializeExecution; + initializeExecution, + + // Tests state in the context of the current execution + testsState = {}; /** * @param {Object} options @@ -214,7 +217,7 @@ module.exports = function (bridge, glob) { var eventId = timers.setEvent(callback); bridge.dispatch(executionRequestEventName, options.cursor, id, eventId, request); - }, dispatchAssertions, new PostmanCookieStore(id, bridge, timers), { + }, testsState, dispatchAssertions, new PostmanCookieStore(id, bridge, timers), { disabledAPIs: initializationOptions.disabledAPIs }) ), diff --git a/lib/sandbox/pmapi-setup-runner.js b/lib/sandbox/pmapi-setup-runner.js index e5ed7186..d33bb7bd 100644 --- a/lib/sandbox/pmapi-setup-runner.js +++ b/lib/sandbox/pmapi-setup-runner.js @@ -1,20 +1,36 @@ +/* eslint-disable function-paren-newline */ +/* eslint-disable one-var */ /** * @fileOverview * * This module externally sets up the test runner on pm api. Essentially, it does not know the insides of pm-api and * does the job completely from outside with minimal external dependency */ -const FUNCTION = 'function'; +const _ = require('lodash'), + FUNCTION = 'function', + uuid = require('../vendor/uuid'), + + OPTIONS = { + When: 'when', + RunCount: 'runCount', + RunUntil: 'runUntil' + }, + OPTION_TYPE = { + [OPTIONS.When]: 'function', + [OPTIONS.RunCount]: 'number', + [OPTIONS.RunUntil]: 'number' + }; /** * @module {PMAPI~setupTestRunner} * @private * * @param {PMAPI} pm - an instance of PM API that it needs - * @param {Function} onAssertionComplete - is the trigger function that is called every time a test is executed and it + * @param {Object} testsState - State of all the tests for the current execution + * @param {Function} onAssertion - is the trigger function that is called every time a test is encountered and it * receives the AssertionInfo object outlining details of the assertion */ -module.exports = function (pm, onAssertionComplete) { +module.exports = function (pm, testsState, onAssertion) { var assertionIndex = 0, /** @@ -23,26 +39,71 @@ module.exports = function (pm, onAssertionComplete) { * @note This is put in a function since this needs to be done from a number of place and having a single * function reduces the chance of bugs * + * @param {String} testId - * @param {String} name - * @param {Boolean} skipped - * * @returns {PMAPI~AssertionInfo} */ - getAssertionObject = function (name, skipped) { + getAssertionObject = function (testId, name, skipped) { /** * @typeDef {AssertionInfo} * @private */ return { + testId: testId, name: String(name), async: false, skipped: Boolean(skipped), passed: true, + pending: !skipped, error: null, index: assertionIndex++ // increment the assertion counter (do it before asserting) }; }, + generateTestId = function (eventName, testName, assertFn, options) { + return [ + eventName, + testName, + assertFn ? assertFn.toString() : '', + JSON.stringify(options) + ].join(''); + }, + + getDefaultTestState = function (options) { + return { + ...(options ? _.pick(options, _.values(OPTIONS)) : {}), + testId: uuid(), + timer: null, + currRunCount: 0, + pending: true + }; + }, + + isOptionConfigured = function (options, optionName) { + return _.has(options, optionName) && typeof options[optionName] === OPTION_TYPE[optionName]; + }, + + + validateOptions = function (options) { + if (!options || typeof options !== 'object') { + throw new Error('Invalid test option: options is not an object'); + } + + const supportedOptions = _.values(OPTIONS); + + Object.keys(options).forEach((optionName) => { + if (!supportedOptions.includes(optionName)) { + throw new Error(`Invalid test option: ${optionName} is not a supported option`); + } + + if (typeof options[optionName] !== OPTION_TYPE[optionName]) { + throw new Error(`Invalid test options: ${optionName} is not a ${OPTION_TYPE[optionName]}`); + } + }); + }, + /** * Simple function to mark an assertion as failed * @@ -57,59 +118,143 @@ module.exports = function (pm, onAssertionComplete) { markAssertionAsFailure = function (assertionData, err) { assertionData.error = err; assertionData.passed = false; + }, + + processAssertion = function (_testId, assertionData, options) { + const testState = testsState[_testId]; + + if (!testState.pending) { + return; + } + + const shouldResolve = Boolean( + assertionData.error || // TODO: Make conditions (test status) to mark a test resolved, configurable. + assertionData.skipped || + _.isEmpty(options) || + !testState || + isOptionConfigured(options, OPTIONS.RunCount) && testState.runCount === testState.currRunCount || + isOptionConfigured(options, OPTIONS.RunUntil) && !testState.timer + ); + + testState.pending = assertionData.pending = !shouldResolve; + + // Tests without options does not need to be tracked + if (_.isEmpty(options)) { + delete testsState[_testId]; + } + + onAssertion(assertionData); + }, + + processOptions = function (_testId, assertionData, options) { + const testState = testsState[_testId], + shouldRun = testState.pending && + (isOptionConfigured(options, OPTIONS.When) ? Boolean(options.when()) : true) && + (isOptionConfigured(options, OPTIONS.RunCount) ? testState.currRunCount < options.runCount : true); + + if (shouldRun) { + testState.currRunCount++; + + const startTimer = isOptionConfigured(options, OPTIONS.RunUntil) && !testState.timer; + + if (startTimer) { + testState.timer = setTimeout(() => { + testState.timer = null; + processAssertion(_testId, assertionData, options); + }, testState.runUntil); + } + } + + return shouldRun; }; /** * @param {String} name - + * @param {Object} [options] - * @param {Function} assert - * @chainable */ - pm.test = function (name, assert) { - var assertionData = getAssertionObject(name, false); + pm.test = function (name, options, assert) { + if (typeof options === FUNCTION) { + assert = options; + options = {}; + } + + if (_.isNil(options) || typeof options !== 'object') { + options = {}; + } + + // TODO: Make generateTestId safe i.e handle invalid `options` as well + const _testId = generateTestId(pm.info.eventName, name, assert, options); + + if (!testsState[_testId]) { + testsState[_testId] = getDefaultTestState(options); + } + + const testState = testsState[_testId], + testId = testState.testId, + assertionData = getAssertionObject(testId, name, false); // if there is no assertion function, we simply move on if (typeof assert !== FUNCTION) { - onAssertionComplete(assertionData); + // Sending `options` as empty to force resolve the test + processAssertion(_testId, assertionData, {}); return pm; } - // if a callback function was sent, then we know that the test is asynchronous - if (assert.length) { - try { - assertionData.async = true; // flag that this was an async test (would be useful later) - - // we execute assertion, but pass it a completion function, which, in turn, raises the completion - // event. we do not need to worry about timers here since we are assuming that some timer within the - // sandbox had actually been the source of async calls and would take care of this - assert(function (err) { - // at first we double check that no synchronous error has happened from the catch block below - if (assertionData.error && assertionData.passed === false) { - return; - } - - // user triggered a failure of the assertion, so we mark it the same - if (err) { - markAssertionAsFailure(assertionData, err); - } - - onAssertionComplete(assertionData); - }); + try { validateOptions(options); } + catch (e) { + markAssertionAsFailure(assertionData, e); + processAssertion(_testId, assertionData, options); + + return pm; + } + + + const shouldRun = processOptions(_testId, assertionData, options); + + if (shouldRun) { + // if a callback function was sent, then we know that the test is asynchronous + if (assert.length) { + try { + assertionData.async = true; // flag that this was an async test (would be useful later) + + // we execute assertion, but pass it a completion function, which, in turn, raises the completion + // event. we do not need to worry about timers here since we are assuming that some timer within the + // sandbox had actually been the source of async calls and would take care of this + assert(function (err) { + // at first we double check that no synchronous error has happened from the catch block below + if (assertionData.error && assertionData.passed === false) { + return; + } + + // user triggered a failure of the assertion, so we mark it the same + if (err) { + markAssertionAsFailure(assertionData, err); + } + + processAssertion(_testId, assertionData, options); + }); + } + // in case a synchronous error occurs in the the async assertion, we still bail out. + catch (e) { + markAssertionAsFailure(assertionData, e); + processAssertion(_testId, assertionData, options); + } } - // in case a synchronous error occurs in the the async assertion, we still bail out. - catch (e) { - markAssertionAsFailure(assertionData, e); - onAssertionComplete(assertionData); + // if the assertion function does not expect a callback, we synchronously execute the same + else { + try { assert(); } + catch (e) { + markAssertionAsFailure(assertionData, e); + } + + processAssertion(_testId, assertionData, options); } } - // if the assertion function does not expect a callback, we synchronously execute the same else { - try { assert(); } - catch (e) { - markAssertionAsFailure(assertionData, e); - } - - onAssertionComplete(assertionData); + processAssertion(_testId, assertionData, options); } return pm; // make it chainable @@ -121,7 +266,7 @@ module.exports = function (pm, onAssertionComplete) { */ pm.test.skip = function (name) { // trigger the assertion events with skips - onAssertionComplete(getAssertionObject(name, true)); + processAssertion(name, getAssertionObject(uuid(), name, true), {}); return pm; // chainable }; diff --git a/lib/sandbox/pmapi.js b/lib/sandbox/pmapi.js index 6e860f37..080311d0 100644 --- a/lib/sandbox/pmapi.js +++ b/lib/sandbox/pmapi.js @@ -44,12 +44,13 @@ const _ = require('lodash'), * * @param {Execution} execution - * @param {Function} onRequest - + * @param {Object} testsState - * @param {Function} onAssertion - * @param {Object} cookieStore - * @param {Object} [options] - * @param {Array.} [options.disabledAPIs] - */ -function Postman (execution, onRequest, onAssertion, cookieStore, options = {}) { +function Postman (execution, onRequest, testsState, onAssertion, cookieStore, options = {}) { // @todo - ensure runtime passes data in a scope format let iterationData = new VariableScope(); @@ -254,7 +255,7 @@ function Postman (execution, onRequest, onAssertion, cookieStore, options = {}) }, options.disabledAPIs); // extend pm api with test runner abilities - setupTestRunner(this, onAssertion); + setupTestRunner(this, testsState, onAssertion); // add response assertions if (this.response) { From a2b5dbaa278647906d4de401264c1a4b5accca28 Mon Sep 17 00:00:00 2001 From: Appurva Murawat Date: Mon, 29 May 2023 11:24:27 +0530 Subject: [PATCH 2/6] Remove global event listeners and abort all pending tests on context dispose. --- lib/postman-sandbox.js | 26 +++++++++++++++++--------- lib/sandbox/execute.js | 8 ++++++++ lib/sandbox/pmapi-setup-runner.js | 6 ++++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/lib/postman-sandbox.js b/lib/postman-sandbox.js index 2b7b9c45..70e307ae 100644 --- a/lib/postman-sandbox.js +++ b/lib/postman-sandbox.js @@ -6,6 +6,8 @@ const _ = require('lodash'), TO_WAIT_BUFFER = 500, // time to wait for sandbox to declare timeout CONSOLE_EVENT_NAME = 'execution.console', + ASSERTION_EVENT_NAME = 'execution.assertion', + ERROR_EVENT_NAME = 'execution.error', EXECUTION_TIMEOUT_ERROR_MESSAGE = 'sandbox not responding', BRIDGE_DISCONNECTING_ERROR_MESSAGE = 'sandbox: execution interrupted, bridge disconnecting.'; @@ -131,19 +133,25 @@ class PostmanSandbox extends UniversalVM { } dispose () { - _.forEach(this._executing, (irq, id) => { - irq && clearTimeout(irq); + this.once('dispose', () => { + _.forEach(this._executing, (irq, id) => { + irq && clearTimeout(irq); - // send an abort event to the sandbox so that it can do cleanups - this.dispatch('execution.abort.' + id); + // send an abort event to the sandbox so that it can do cleanups + this.dispatch('execution.abort.' + id); - // even though sandbox could bubble the result event upon receiving abort, that would reduce - // stability of the system in case sandbox was unresponsive. - this.emit('execution.result.' + id, new Error(BRIDGE_DISCONNECTING_ERROR_MESSAGE)); + // even though sandbox could bubble the result event upon receiving abort, that would reduce + // stability of the system in case sandbox was unresponsive. + this.emit('execution.result.' + id, new Error(BRIDGE_DISCONNECTING_ERROR_MESSAGE)); + }); + + this.removeAllListeners(CONSOLE_EVENT_NAME); + this.removeAllListeners(ASSERTION_EVENT_NAME); + this.removeAllListeners(ERROR_EVENT_NAME); + this.disconnect(); }); - this.removeAllListeners(CONSOLE_EVENT_NAME); - this.disconnect(); + this.dispatch('dispose'); } } diff --git a/lib/sandbox/execute.js b/lib/sandbox/execute.js index 66520d3d..85bf2d93 100644 --- a/lib/sandbox/execute.js +++ b/lib/sandbox/execute.js @@ -86,6 +86,14 @@ module.exports = function (bridge, glob) { }); }); + bridge.once('dispose', () => { + // Abort all pending assertions and cleanup the global tests state + Object.values(testsState).forEach((test) => { test.abort(); }); + testsState = {}; + + bridge.dispatch('dispose'); + }); + /** * @param {String} id * @param {Event} event diff --git a/lib/sandbox/pmapi-setup-runner.js b/lib/sandbox/pmapi-setup-runner.js index d33bb7bd..a281cbd3 100644 --- a/lib/sandbox/pmapi-setup-runner.js +++ b/lib/sandbox/pmapi-setup-runner.js @@ -195,6 +195,12 @@ module.exports = function (pm, testsState, onAssertion) { testId = testState.testId, assertionData = getAssertionObject(testId, name, false); + // TODO: Do this along with test state initialization. + testState.abort = () => { + markAssertionAsFailure(assertionData, new Error('Execution aborted before test could complete')); + processAssertion(_testId, assertionData, options); + }; + // if there is no assertion function, we simply move on if (typeof assert !== FUNCTION) { // Sending `options` as empty to force resolve the test From 165c20be629e94511e106864c625b40f4dd083cf Mon Sep 17 00:00:00 2001 From: Appurva Murawat Date: Mon, 29 May 2023 11:57:45 +0530 Subject: [PATCH 3/6] Add callback in `dispose` --- lib/postman-sandbox-fleet.js | 17 ++++++++++++++--- lib/postman-sandbox.js | 4 +++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/postman-sandbox-fleet.js b/lib/postman-sandbox-fleet.js index 6a2afc50..f01ad9a3 100644 --- a/lib/postman-sandbox-fleet.js +++ b/lib/postman-sandbox-fleet.js @@ -159,13 +159,24 @@ class PostmanSandboxFleet { /** * Dispose off all initialized sandbox instances from the fleet * + * @param {Function} [callback] - * @returns {void} */ - disposeAll () { + disposeAll (callback) { + let disposedCount = 0; + + if (typeof callback !== 'function') { + callback = _.noop; + } + this.fleet.forEach((context, templateName) => { - context.dispose(); + context.dispose(() => { + this.fleet.delete(templateName); - this.fleet.delete(templateName); + if (++disposedCount === this.fleet.size) { + return callback(); + } + }); }); } } diff --git a/lib/postman-sandbox.js b/lib/postman-sandbox.js index 70e307ae..71a91c8c 100644 --- a/lib/postman-sandbox.js +++ b/lib/postman-sandbox.js @@ -132,7 +132,7 @@ class PostmanSandbox extends UniversalVM { }); } - dispose () { + dispose (callback) { this.once('dispose', () => { _.forEach(this._executing, (irq, id) => { irq && clearTimeout(irq); @@ -149,6 +149,8 @@ class PostmanSandbox extends UniversalVM { this.removeAllListeners(ASSERTION_EVENT_NAME); this.removeAllListeners(ERROR_EVENT_NAME); this.disconnect(); + + typeof callback === 'function' && callback(); }); this.dispatch('dispose'); From 1a479bc653c12b51dfd034965227124b571e9f79 Mon Sep 17 00:00:00 2001 From: Appurva Murawat Date: Mon, 29 May 2023 16:00:43 +0530 Subject: [PATCH 4/6] Fix skipped test execution --- lib/sandbox/pmapi-setup-runner.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/sandbox/pmapi-setup-runner.js b/lib/sandbox/pmapi-setup-runner.js index a281cbd3..6e968412 100644 --- a/lib/sandbox/pmapi-setup-runner.js +++ b/lib/sandbox/pmapi-setup-runner.js @@ -123,6 +123,10 @@ module.exports = function (pm, testsState, onAssertion) { processAssertion = function (_testId, assertionData, options) { const testState = testsState[_testId]; + if (!testState) { + return onAssertion(assertionData); + } + if (!testState.pending) { return; } @@ -131,9 +135,8 @@ module.exports = function (pm, testsState, onAssertion) { assertionData.error || // TODO: Make conditions (test status) to mark a test resolved, configurable. assertionData.skipped || _.isEmpty(options) || - !testState || isOptionConfigured(options, OPTIONS.RunCount) && testState.runCount === testState.currRunCount || - isOptionConfigured(options, OPTIONS.RunUntil) && !testState.timer + isOptionConfigured(options, OPTIONS.RunUntil) && testState.currRunCount && !testState.timer ); testState.pending = assertionData.pending = !shouldResolve; From 638579895f5b20c3302b096cd4a838b6d3766792 Mon Sep 17 00:00:00 2001 From: Appurva Murawat Date: Mon, 5 Jun 2023 11:07:05 +0530 Subject: [PATCH 5/6] Modify behavior of `runUntil` test option --- lib/sandbox/pmapi-setup-runner.js | 46 ++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/lib/sandbox/pmapi-setup-runner.js b/lib/sandbox/pmapi-setup-runner.js index 6e968412..06dd08da 100644 --- a/lib/sandbox/pmapi-setup-runner.js +++ b/lib/sandbox/pmapi-setup-runner.js @@ -12,13 +12,13 @@ const _ = require('lodash'), OPTIONS = { When: 'when', - RunCount: 'runCount', - RunUntil: 'runUntil' + Count: 'count', + Timeout: 'timeout' }, OPTION_TYPE = { [OPTIONS.When]: 'function', - [OPTIONS.RunCount]: 'number', - [OPTIONS.RunUntil]: 'number' + [OPTIONS.Count]: 'number', + [OPTIONS.Timeout]: 'number' }; /** @@ -76,7 +76,7 @@ module.exports = function (pm, testsState, onAssertion) { ...(options ? _.pick(options, _.values(OPTIONS)) : {}), testId: uuid(), timer: null, - currRunCount: 0, + runCount: 0, pending: true }; }, @@ -120,6 +120,8 @@ module.exports = function (pm, testsState, onAssertion) { assertionData.passed = false; }, + // Processes the assertion data and test state to determine if the assertion should be resolved or not + // and then calls the onAssertion callback processAssertion = function (_testId, assertionData, options) { const testState = testsState[_testId]; @@ -135,8 +137,8 @@ module.exports = function (pm, testsState, onAssertion) { assertionData.error || // TODO: Make conditions (test status) to mark a test resolved, configurable. assertionData.skipped || _.isEmpty(options) || - isOptionConfigured(options, OPTIONS.RunCount) && testState.runCount === testState.currRunCount || - isOptionConfigured(options, OPTIONS.RunUntil) && testState.currRunCount && !testState.timer + isOptionConfigured(options, OPTIONS.Count) && testState.count === testState.runCount || + isOptionConfigured(options, OPTIONS.Timeout) && testState.runCount && !testState.timer ); testState.pending = assertionData.pending = !shouldResolve; @@ -149,23 +151,35 @@ module.exports = function (pm, testsState, onAssertion) { onAssertion(assertionData); }, + + // Processes the provided options and updates the test state accordingly every time a test spec is encountered processOptions = function (_testId, assertionData, options) { const testState = testsState[_testId], shouldRun = testState.pending && (isOptionConfigured(options, OPTIONS.When) ? Boolean(options.when()) : true) && - (isOptionConfigured(options, OPTIONS.RunCount) ? testState.currRunCount < options.runCount : true); + (isOptionConfigured(options, OPTIONS.Count) ? testState.runCount < options.count : true), + startTimer = + isOptionConfigured(options, OPTIONS.Timeout) && + testState.runCount === 0 && + !testState.timer; if (shouldRun) { - testState.currRunCount++; + testState.runCount++; + } - const startTimer = isOptionConfigured(options, OPTIONS.RunUntil) && !testState.timer; + if (startTimer) { + testState.timer = setTimeout(() => { + const shouldFail = + (!testState.count && testState.runCount === 0) || + (isOptionConfigured(options, OPTIONS.Count) && testState.runCount < options.count); - if (startTimer) { - testState.timer = setTimeout(() => { - testState.timer = null; - processAssertion(_testId, assertionData, options); - }, testState.runUntil); - } + if (shouldFail) { + markAssertionAsFailure(assertionData, new Error('Test timed out')); + } + + testState.timer = null; + processAssertion(_testId, assertionData, options); + }, testState.timeout); } return shouldRun; From 5d56bbcc9778956e817743aa49e010176e5aa600 Mon Sep 17 00:00:00 2001 From: Appurva Murawat Date: Tue, 6 Jun 2023 14:55:04 +0530 Subject: [PATCH 6/6] Generate test id from internal stack trace --- lib/sandbox/pmapi-setup-runner.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/sandbox/pmapi-setup-runner.js b/lib/sandbox/pmapi-setup-runner.js index 06dd08da..7c0798fa 100644 --- a/lib/sandbox/pmapi-setup-runner.js +++ b/lib/sandbox/pmapi-setup-runner.js @@ -8,6 +8,7 @@ */ const _ = require('lodash'), FUNCTION = 'function', + CryptoJS = require('crypto-js'), uuid = require('../vendor/uuid'), OPTIONS = { @@ -62,13 +63,12 @@ module.exports = function (pm, testsState, onAssertion) { }; }, - generateTestId = function (eventName, testName, assertFn, options) { - return [ - eventName, - testName, - assertFn ? assertFn.toString() : '', - JSON.stringify(options) - ].join(''); + generateTestId = function () { + const stackTrace = new Error('_internal_').stack; + + // We are creating a hash of the stack trace to generate a unique test id. + // This is done to uniquely identify each test in a script irrespective of its signature + return CryptoJS.SHA256(stackTrace).toString(); }, getDefaultTestState = function (options) {