Allow user to specify a return code when the test fails

packages/cli/src/cmd/agent/start.ts

@@ -1,7 +1,6 @@
 import { CommandModule, Arguments } from 'yargs'
 import { checkFile } from '../common'
 import run from '@flood/element-flood-runner'
-import { runUntilExit } from '@flood/element-api'
 
 interface RunArguments extends Arguments {
 	file: string
@@ -14,7 +13,7 @@ const cmd: CommandModule = {
 	handler(args: RunArguments) {
 		const { file } = args
 
-		return runUntilExit(() => run(file))
+		run(file)
 	},
 
 	builder(yargs) {

packages/cli/src/cmd/run.ts

@@ -1,7 +1,6 @@
 /* eslint-disable @typescript-eslint/no-use-before-define */
 import {
 	runCommandLine,
-	runUntilExit,
 	ElementOptions,
 	WorkRoot,
 	FloodProcessEnv,
@@ -32,6 +31,7 @@ interface RunArguments extends Arguments {
 	slowMo?: number
 	'work-root'?: string
 	'test-data-root'?: string
+	'fail-status-code': number
 }
 
 function setupDelayOverrides(args: RunArguments, testSettingOverrides: TestSettings) {
@@ -85,6 +85,7 @@ const cmd: CommandModule = {
 			runEnv: initRunEnv(workRootPath, testDataPath),
 			testSettingOverrides: {},
 			persistentRunner: false,
+			failStatusCode: args['fail-status-code'],
 		}
 
 		if (args.loopCount) {
@@ -99,7 +100,7 @@ const cmd: CommandModule = {
 			opts.testCommander = makeTestCommander(file)
 		}
 
-		runUntilExit(() => runCommandLine(opts))
+		runCommandLine(opts)
 	},
 	builder(yargs: Argv): Argv {
 		return yargs
@@ -188,6 +189,11 @@ const cmd: CommandModule = {
 			.option('verbose', {
 				describe: 'Verbose mode',
 			})
+			.option('fail-status-code', {
+				describe: 'Exit code when the test fails',
+				type: 'number',
+				default: 1,
+			})
 			.positional('file', {
 				describe: 'the test script to run',
 			})

packages/core/api.ts

@@ -10,7 +10,7 @@ export {
 
 // XYZ
 
-export { runCommandLine, runUntilExit, ElementOptions } from './src/Element'
+export { runCommandLine, ElementOptions } from './src/Element'
 
 export { RuntimeEnvironment } from './src/runtime-environment/types'
 export { nullRuntimeEnvironment } from './src/runtime-environment/NullRuntimeEnvironment'

packages/core/src/Element.ts

@@ -27,18 +27,7 @@ export interface ElementOptions {
 	testObserverFactory?: (t: TestObserver) => TestObserver
 	persistentRunner: boolean
 	testCommander?: TestCommander
-}
-
-export function runUntilExit(fn: () => Promise<void>) {
-	fn()
-		.then(() => {
-			process.exit(0)
-		})
-		.catch(err => {
-			console.log('Element exited with error')
-			console.error(err)
-			process.exit(1)
-		})
+	failStatusCode: number
 }
 
 export async function runCommandLine(opts: ElementOptions): Promise<void> {
@@ -95,5 +84,12 @@ export async function runCommandLine(opts: ElementOptions): Promise<void> {
 		return new EvaluatedScript(opts.runEnv, await mustCompileFile(testScript, testScriptOptions))
 	}
 
-	await runner.run(testScriptFactory)
+	try {
+		await runner.run(testScriptFactory)
+		process.exit(0)
+	} catch (err) {
+		console.log('Element exited with error')
+		console.error(err)
+		process.exit(opts.failStatusCode)
+	}
 }

packages/core/src/Runner.ts

@@ -193,7 +193,6 @@ export class Runner {
 			})
 
 			this.logger.info(`Test completed after ${this.looper.iterations} iterations`)
-			return
 		} catch (err) {
 			if (err instanceof TestScriptError) {
 				this.logger.error('\n' + err.toStringNodeFormat())
@@ -205,6 +204,7 @@ export class Runner {
 			// if (process.env.NODE_ENV !== 'production') {
 			this.logger.debug(err.stack)
 			// }
+			throw err
 		}
 
 		if (testToCancel !== undefined) {

packages/core/src/runtime/Test.ts

@@ -158,7 +158,7 @@ export default class Test implements ITest {
 
 				if (this.failed) {
 					console.log('failed, bailing out of steps')
-					break
+					throw Error('test failed')
 				}
 			}
 		} catch (err) {

packages/element-api/index.ts

@@ -10,7 +10,7 @@ export {
 
 // XYZ
 
-export { runCommandLine, runUntilExit, ElementOptions } from '@flood/element-core'
+export { runCommandLine, ElementOptions } from '@flood/element-core'
 
 export { TestSettings, ResponseTiming } from '@flood/element-core'
 export { RuntimeEnvironment, FloodProcessEnv } from '@flood/element-core'

packages/flood-runner/src/Grid.ts

@@ -32,6 +32,7 @@ export async function run(file: string): Promise<void> {
 		testSettingOverrides: {},
 		testObserverFactory,
 		persistentRunner: false,
+		failStatusCode: 1,
 	}
 
 	if (gridConfig.testDuration !== undefined) {