Skip to content

Commit

Permalink
Add support for asynchronous tests with pending state.
Browse files Browse the repository at this point in the history
  • Loading branch information
@appurva21
appurva21 committed May 23, 2023
1 parent f1665e5 commit af5c196
Show file tree
Hide file tree
Showing 3 changed files with 200 additions and 45 deletions.
20 changes: 17 additions & 3 deletions lib/sandbox/execute.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ const _ = require('lodash'),
EXECUTION_ASSERTION_EVENT = 'execution.assertion',
EXECUTION_ASSERTION_EVENT_BASE = 'execution.assertion.',

TESTS_RESOLVED_EVENT = 'tests.resolved',

executeContext = require('./execute-context');

module.exports = function (bridge, glob) {
Expand All @@ -34,7 +36,10 @@ module.exports = function (bridge, glob) {
// For caching required information provided during
// initialization which will be used during execution
let initializationOptions = {},
initializeExecution;
initializeExecution,

// Tests state in the context of the current execution
testsState = {};

/**
* @param {Object} options
Expand Down Expand Up @@ -129,15 +134,24 @@ module.exports = function (bridge, glob) {
* @param {Boolean} assertions[].async -
* @param {Boolean} assertions[].passed -
* @param {Boolean} assertions[].skipped -
* @param {Boolean} resolved -
*/
dispatchAssertions = function (assertions) {
dispatchAssertions = function (assertions, resolved) {
// Legacy `test` API accumulates all the assertions and dispatches at once
// whereas, `pm.test` dispatch on every single assertion.
// For compatibility, dispatch the single assertion as an array.
!Array.isArray(assertions) && (assertions = [assertions]);

bridge.dispatch(assertionEventName, options.cursor, assertions);
bridge.dispatch(EXECUTION_ASSERTION_EVENT, options.cursor, assertions);

const dispatchAllTestsResolved =
resolved &&
Object.values(testsState).every((v) => { return v.resolved; });

if (dispatchAllTestsResolved) {
bridge.dispatch(TESTS_RESOLVED_EVENT);
}
};

let waiting,
Expand Down Expand Up @@ -214,7 +228,7 @@ module.exports = function (bridge, glob) {
var eventId = timers.setEvent(callback);

bridge.dispatch(executionRequestEventName, options.cursor, id, eventId, request);
}, dispatchAssertions, new PostmanCookieStore(id, bridge, timers), {
}, testsState, dispatchAssertions, new PostmanCookieStore(id, bridge, timers), {
disabledAPIs: initializationOptions.disabledAPIs
})
),
Expand Down
220 changes: 180 additions & 40 deletions lib/sandbox/pmapi-setup-runner.js
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,35 @@
/* eslint-disable one-var */
/**
* @fileOverview
*
* This module externally sets up the test runner on pm api. Essentially, it does not know the insides of pm-api and
* does the job completely from outside with minimal external dependency
*/
const FUNCTION = 'function';
const _ = require('lodash'),
FUNCTION = 'function',
uuid = require('../vendor/uuid'),

OPTIONS = {
When: 'when',
RunCount: 'runCount',
RunUntil: 'runUntil'
},
OPTION_TYPE = {
[OPTIONS.When]: 'function',
[OPTIONS.RunCount]: 'number',
[OPTIONS.RunUntil]: 'number'
};

/**
* @module {PMAPI~setupTestRunner}
* @private
*
* @param {PMAPI} pm - an instance of PM API that it needs
* @param {Function} onAssertionComplete - is the trigger function that is called every time a test is executed and it
* @param {Object} testsState - State of all the tests for the current execution
* @param {Function} onAssertion - is the trigger function that is called every time a test is encountered and it
* receives the AssertionInfo object outlining details of the assertion
*/
module.exports = function (pm, onAssertionComplete) {
module.exports = function (pm, testsState, onAssertion) {
var assertionIndex = 0,

/**
Expand All @@ -25,24 +40,73 @@ module.exports = function (pm, onAssertionComplete) {
*
* @param {String} name -
* @param {Boolean} skipped -
* @param {String} [id] -
*
* @returns {PMAPI~AssertionInfo}
*/
getAssertionObject = function (name, skipped) {
getAssertionObject = function (name, skipped, id) {
if (!id) {
id = uuid();
}

/**
* @typeDef {AssertionInfo}
* @private
*/
return {
id: id,
name: String(name),
async: false,
skipped: Boolean(skipped),
passed: true,
pending: !skipped,
error: null,
index: assertionIndex++ // increment the assertion counter (do it before asserting)
};
},

generateTestId = function (eventName, testName, assertFn, options) {
return [
eventName,
testName,
assertFn ? assertFn.toString() : '',
JSON.stringify(options)
].join('');
},

getDefaultTestState = function (options) {
return {
...(options ? _.pick(options, _.values(OPTIONS)) : {}),
id: uuid(),
timer: null,
currRunCount: 0,
resolved: false
};
},

isOptionConfigured = function (options, optionName) {
return _.has(options, optionName) && typeof options[optionName] === OPTION_TYPE[optionName];
},


validateOptions = function (options) {
if (!options || typeof options !== 'object') {
throw new Error('Invalid test option: options is not an object');
}

const supportedOptions = _.values(OPTIONS);

Object.keys(options).forEach((optionName) => {
if (!supportedOptions.includes(optionName)) {
throw new Error(`Invalid test option: ${optionName} is not a supported option`);
}

if (typeof options[optionName] !== OPTION_TYPE[optionName]) {
throw new Error(`Invalid test options: ${optionName} is not a ${OPTION_TYPE[optionName]}`);
}
});
},

/**
* Simple function to mark an assertion as failed
*
Expand All @@ -57,59 +121,135 @@ module.exports = function (pm, onAssertionComplete) {
markAssertionAsFailure = function (assertionData, err) {
assertionData.error = err;
assertionData.passed = false;
},

processAssertion = function (_testId, assertionData, options) {
const testState = testsState[_testId];

if (testState.resolved) {
return;
}

const shouldResolve =
assertionData.error || // TODO: Make conditions (test status) to mark a test resolved, configurable.
assertionData.skipped ||
_.isEmpty(options) ||
!testState ||
isOptionConfigured(options, OPTIONS.RunCount) && testState.runCount === testState.currRunCount ||
isOptionConfigured(options, OPTIONS.RunUntil) && !testState.timer;


testState.resolved = shouldResolve;
assertionData.pending = !shouldResolve;

onAssertion(assertionData, shouldResolve);
},

updateTestState = function (_testId, assertionData, options) {
const testState = testsState[_testId],
startTimer = !testState.timer && isOptionConfigured(options, OPTIONS.RunUntil);

testState.currRunCount++;

if (startTimer) {
testState.timer = setTimeout(() => {
testState.timer = null;
processAssertion(_testId, assertionData, options);
}, testState.runUntil);
}
};

/**
* @param {String} name -
* @param {Object} [options] -
* @param {Function} assert -
* @chainable
*/
pm.test = function (name, assert) {
var assertionData = getAssertionObject(name, false);
pm.test = function (name, options, assert) {
if (typeof options === FUNCTION) {
assert = options;
options = {};
}

if (_.isNil(options) || typeof options !== 'object') {
options = {};
}

// TODO: Make generateTestId safe i.e handle invalid `options` as well
const _testId = generateTestId(pm.info.eventName, name, assert, options);

if (!testsState[_testId]) {
testsState[_testId] = getDefaultTestState(options);
}

const testState = testsState[_testId],
testId = testState.testId,
assertionData = getAssertionObject(name, false, testId);

// if there is no assertion function, we simply move on
if (typeof assert !== FUNCTION) {
onAssertionComplete(assertionData);
// Sending `options` as empty to force resolve the test
processAssertion(_testId, assertionData, {});

return pm;
}

// if a callback function was sent, then we know that the test is asynchronous
if (assert.length) {
try {
assertionData.async = true; // flag that this was an async test (would be useful later)

// we execute assertion, but pass it a completion function, which, in turn, raises the completion
// event. we do not need to worry about timers here since we are assuming that some timer within the
// sandbox had actually been the source of async calls and would take care of this
assert(function (err) {
// at first we double check that no synchronous error has happened from the catch block below
if (assertionData.error && assertionData.passed === false) {
return;
}

// user triggered a failure of the assertion, so we mark it the same
if (err) {
markAssertionAsFailure(assertionData, err);
}

onAssertionComplete(assertionData);
});
try { validateOptions(options); }
catch (e) {
markAssertionAsFailure(assertionData, e);
processAssertion(_testId, assertionData, options);

return pm;
}

const shouldRun =
!testState.resolved &&
(isOptionConfigured(options, OPTIONS.When) ? Boolean(options.when()) : true) &&
(isOptionConfigured(options, OPTIONS.RunCount) ? testState.currRunCount < options.runCount : true);

if (shouldRun) {
updateTestState(_testId, assertionData, options);

// if a callback function was sent, then we know that the test is asynchronous
if (assert.length) {
try {
assertionData.async = true; // flag that this was an async test (would be useful later)

// we execute assertion, but pass it a completion function, which, in turn, raises the completion
// event. we do not need to worry about timers here since we are assuming that some timer within the
// sandbox had actually been the source of async calls and would take care of this
assert(function (err) {
// at first we double check that no synchronous error has happened from the catch block below
if (assertionData.error && assertionData.passed === false) {
return;
}

// user triggered a failure of the assertion, so we mark it the same
if (err) {
markAssertionAsFailure(assertionData, err);
}

processAssertion(_testId, assertionData, options);
});
}
// in case a synchronous error occurs in the the async assertion, we still bail out.
catch (e) {
markAssertionAsFailure(assertionData, e);
processAssertion(_testId, assertionData, options);
}
}
// in case a synchronous error occurs in the the async assertion, we still bail out.
catch (e) {
markAssertionAsFailure(assertionData, e);
onAssertionComplete(assertionData);
// if the assertion function does not expect a callback, we synchronously execute the same
else {
try { assert(); }
catch (e) {
markAssertionAsFailure(assertionData, e);
}

processAssertion(_testId, assertionData, options);
}
}
// if the assertion function does not expect a callback, we synchronously execute the same
else {
try { assert(); }
catch (e) {
markAssertionAsFailure(assertionData, e);
}

onAssertionComplete(assertionData);
processAssertion(_testId, assertionData, options);
}

return pm; // make it chainable
Expand All @@ -121,7 +261,7 @@ module.exports = function (pm, onAssertionComplete) {
*/
pm.test.skip = function (name) {
// trigger the assertion events with skips
onAssertionComplete(getAssertionObject(name, true));
processAssertion(name, getAssertionObject(name, true), {});

return pm; // chainable
};
Expand Down
5 changes: 3 additions & 2 deletions lib/sandbox/pmapi.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,13 @@ const _ = require('lodash'),
*
* @param {Execution} execution -
* @param {Function} onRequest -
* @param {Object} testsState -
* @param {Function} onAssertion -
* @param {Object} cookieStore -
* @param {Object} [options] -
* @param {Array.<String>} [options.disabledAPIs] -
*/
function Postman (execution, onRequest, onAssertion, cookieStore, options = {}) {
function Postman (execution, onRequest, testsState, onAssertion, cookieStore, options = {}) {
// @todo - ensure runtime passes data in a scope format
let iterationData = new VariableScope();

Expand Down Expand Up @@ -254,7 +255,7 @@ function Postman (execution, onRequest, onAssertion, cookieStore, options = {})
}, options.disabledAPIs);

// extend pm api with test runner abilities
setupTestRunner(this, onAssertion);
setupTestRunner(this, testsState, onAssertion);

// add response assertions
if (this.response) {
Expand Down

0 comments on commit af5c196

Please sign in to comment.