From 964ff8ba8ffff16c068f31227605ce7b433bc368 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Tue, 27 Jul 2021 21:43:56 -0500 Subject: [PATCH 1/3] Add --quiet-exit-code and make it default --rich-events behavior Fixes https://github.com/elastic/synthetics/issues/356 Accomplishes the fix by making --rich-events always exit with a code of 0, even if tests fail --- __tests__/cli.test.ts | 12 ++++++++++++ __tests__/index.test.ts | 1 + src/cli.ts | 14 ++++++++------ src/common_types.ts | 1 + src/parse_args.ts | 5 +++++ 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/__tests__/cli.test.ts b/__tests__/cli.test.ts index 46dd9418..a169c3b2 100644 --- a/__tests__/cli.test.ts +++ b/__tests__/cli.test.ts @@ -41,6 +41,17 @@ describe('CLI', () => { expect(await cli.exitCode).toBe(1); }); + describe('with --quiet-exit-code', () => { + it('runs the suites, exiting with 0', async () => { + const cli = new CLIMock([ + join(FIXTURES_DIR, 'error.journey.ts'), + '--quiet-exit-code', + ]); + await cli.waitFor('boom'); + expect(await cli.exitCode).toBe(0); + }); + }); + it('produce json output --json and reporter=json flag', async () => { const output = async args => { const cli = new CLIMock([join(FIXTURES_DIR, 'fake.journey.ts'), ...args]); @@ -77,6 +88,7 @@ describe('CLI', () => { it('mimick new heartbeat with `--rich-events` flag', async () => { const cli = new CLIMock([ join(FIXTURES_DIR, 'fake.journey.ts'), + join(FIXTURES_DIR, 'error.journey.ts'), '--rich-events', ]); await cli.waitFor('journey/end'); diff --git a/__tests__/index.test.ts b/__tests__/index.test.ts index 140e959a..8fdbed28 100644 --- a/__tests__/index.test.ts +++ b/__tests__/index.test.ts @@ -72,6 +72,7 @@ describe('Run', () => { match: 'check*', network: true, pauseOnError: true, + quietExitCode: true, reporter: 'json', }; await run(options); diff --git a/src/cli.ts b/src/cli.ts index e31c8f6d..f19a692b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -194,12 +194,14 @@ async function prepareSuites(inputs: string[]) { ...options, }); - /** - * Exit with error status if any journey fails - */ - for (const result of Object.values(results)) { - if (result.status === 'failed') { - process.exit(1); + if (!options.quietExitCode) { + /** + * Exit with error status if any journey fails + */ + for (const result of Object.values(results)) { + if (result.status === 'failed') { + process.exit(1); + } } } })().catch(e => { diff --git a/src/common_types.ts b/src/common_types.ts index 950c0969..c0963547 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -130,6 +130,7 @@ export type CliArgs = { dryRun?: boolean; network?: boolean; pauseOnError?: boolean; + quietExitCode?: boolean; reporter?: Reporters; wsEndpoint?: string; sandbox?: boolean; diff --git a/src/parse_args.ts b/src/parse_args.ts index 07824e90..36f578f5 100644 --- a/src/parse_args.ts +++ b/src/parse_args.ts @@ -98,6 +98,10 @@ program '--pause-on-error', 'pause on error until a keypress is made in the console. Useful during development' ) + .option( + '--quiet-exit-code', + 'always return 0 as an exit code status, regardless of test pass / fail. Only return > 0 exit codes on internal errors where the suite could not be run' + ) .version(version) .description('Run synthetic tests'); @@ -112,6 +116,7 @@ if (options.richEvents) { options.reporter = options.reporter ?? 'json'; options.ssblocks = true; options.network = true; + options.quietExitCode = true; } if (options.capability) { From 885a0ea0eb3141686f2f2538308a1092f2fe7eee Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Wed, 28 Jul 2021 15:29:34 -0500 Subject: [PATCH 2/3] Incorporate PR Feedback --- __tests__/cli.test.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/__tests__/cli.test.ts b/__tests__/cli.test.ts index a169c3b2..f9a1de60 100644 --- a/__tests__/cli.test.ts +++ b/__tests__/cli.test.ts @@ -41,15 +41,13 @@ describe('CLI', () => { expect(await cli.exitCode).toBe(1); }); - describe('with --quiet-exit-code', () => { - it('runs the suites, exiting with 0', async () => { - const cli = new CLIMock([ - join(FIXTURES_DIR, 'error.journey.ts'), - '--quiet-exit-code', - ]); - await cli.waitFor('boom'); - expect(await cli.exitCode).toBe(0); - }); + it('runs the suites with --quiet-exit-code, always exiting with 0', async () => { + const cli = new CLIMock([ + join(FIXTURES_DIR, 'error.journey.ts'), + '--quiet-exit-code', + ]); + await cli.waitFor('boom'); + expect(await cli.exitCode).toBe(0); }); it('produce json output --json and reporter=json flag', async () => { @@ -106,7 +104,7 @@ describe('CLI', () => { }); expect(await cli.exitCode).toBe(0); - }); + }, 30); it('override screenshots with `--rich-events` flag', async () => { const cli = new CLIMock([ From 060ca5904c7e226368794a829a9e14557c691cda Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 29 Jul 2021 08:44:17 -0500 Subject: [PATCH 3/3] Fix millis --- __tests__/cli.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/cli.test.ts b/__tests__/cli.test.ts index f9a1de60..d0736861 100644 --- a/__tests__/cli.test.ts +++ b/__tests__/cli.test.ts @@ -104,7 +104,7 @@ describe('CLI', () => { }); expect(await cli.exitCode).toBe(0); - }, 30); + }, 30000); it('override screenshots with `--rich-events` flag', async () => { const cli = new CLIMock([