From 5586c258a87285006349d2a5cf8558502c515211 Mon Sep 17 00:00:00 2001 From: tommy-mitchell Date: Tue, 14 Mar 2023 00:18:00 -0500 Subject: [PATCH 1/8] fix: don't log stacktrace if cli succeeds --- source/cli.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/cli.ts b/source/cli.ts index c6dbed5f..0587cb43 100644 --- a/source/cli.ts +++ b/source/cli.ts @@ -54,14 +54,17 @@ const cli = meow(` const diagnostics = await tsd(options); if (diagnostics.length > 0) { - throw new Error(formatter(diagnostics, showDiff)); + throw new Error(formatter(diagnostics, showDiff), {cause: 'tsd found diagnostics'}); } } catch (error: unknown) { const potentialError = error as Error | undefined; - const errorMessage = potentialError?.stack ?? potentialError?.message; - if (errorMessage) { - console.error(`Error running tsd: ${errorMessage}`); + if (potentialError?.cause === 'tsd found diagnostics') { + if (potentialError?.message) { + console.error(potentialError?.message); + } + } else if (potentialError?.stack) { + console.error(`Error running tsd: ${potentialError?.stack}`); } process.exit(1); From 0a92e2ebd7a21ed053e08cf6a713f6bbf38b2e79 Mon Sep 17 00:00:00 2001 From: tommy-mitchell Date: Tue, 14 Mar 2023 00:23:52 -0500 Subject: [PATCH 2/8] fix: new line on failure --- source/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/cli.ts b/source/cli.ts index 0587cb43..7d8e0215 100644 --- a/source/cli.ts +++ b/source/cli.ts @@ -64,7 +64,7 @@ const cli = meow(` console.error(potentialError?.message); } } else if (potentialError?.stack) { - console.error(`Error running tsd: ${potentialError?.stack}`); + console.error(`Error running tsd:\n${potentialError?.stack}`); } process.exit(1); From 4e1dc85d961c3f2fe9f25be361c04a9dd827e0dd Mon Sep 17 00:00:00 2001 From: tommy-mitchell Date: Tue, 14 Mar 2023 00:27:00 -0500 Subject: [PATCH 3/8] tests: add `verifyCli` utility to check results --- source/test/cli.ts | 55 ++++++++++++++++++++++++++++------- source/test/diff.ts | 15 ++-------- source/test/fixtures/utils.ts | 24 ++++++++++++++- 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/source/test/cli.ts b/source/test/cli.ts index a549d98b..891fbe25 100644 --- a/source/test/cli.ts +++ b/source/test/cli.ts @@ -3,6 +3,7 @@ import test from 'ava'; import execa from 'execa'; import readPkgUp from 'read-pkg-up'; import tsd, {formatter} from '..'; +import {verifyCli} from './fixtures/utils'; interface ExecaError extends Error { readonly exitCode: number; @@ -15,7 +16,11 @@ test('fail if errors are found', async t => { })); t.is(exitCode, 1); - t.regex(stderr, /5:19[ ]{2}Argument of type number is not assignable to parameter of type string./); + verifyCli(t, stderr, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); }); test('succeed if no errors are found', async t => { @@ -32,7 +37,11 @@ test('provide a path', async t => { const {exitCode, stderr} = await t.throwsAsync(execa('dist/cli.js', [file])); t.is(exitCode, 1); - t.regex(stderr, /5:19[ ]{2}Argument of type number is not assignable to parameter of type string./); + verifyCli(t, stderr, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); }); test('cli help flag', async t => { @@ -57,7 +66,11 @@ test('cli typings flag', async t => { })); t.is(exitCode, 1); - t.true(stderr.includes('✖ 5:19 Argument of type number is not assignable to parameter of type string.')); + verifyCli(t, stderr, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); }; await runTest('--typings'); @@ -71,7 +84,11 @@ test('cli files flag', async t => { })); t.is(exitCode, 1); - t.true(stderr.includes('✖ 5:19 Argument of type number is not assignable to parameter of type string.')); + verifyCli(t, stderr, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); }; await runTest('--files'); @@ -84,7 +101,11 @@ test('cli files flag array', async t => { })); t.is(exitCode, 1); - t.true(stderr.includes('✖ 5:19 Argument of type number is not assignable to parameter of type string.')); + verifyCli(t, stderr, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); }); test('cli typings and files flags', async t => { @@ -94,17 +115,27 @@ test('cli typings and files flags', async t => { const {exitCode, stderr} = t.throws(() => execa.commandSync(`dist/cli.js -t ${typingsFile} -f ${testFile}`)); t.is(exitCode, 1); - t.true(stderr.includes('✖ 5:19 Argument of type number is not assignable to parameter of type string.')); + verifyCli(t, stderr, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); }); test('tsd logs stacktrace on failure', async t => { - const {exitCode, stderr, stack} = await t.throwsAsync(execa('../../../cli.js', { + const {exitCode, stderr} = await t.throwsAsync(execa('../../../cli.js', { cwd: path.join(__dirname, 'fixtures/empty-package-json') })); t.is(exitCode, 1); - t.true(stderr.includes('Error running tsd: JSONError: Unexpected end of JSON input while parsing empty string')); - t.truthy(stack); + + verifyCli(t, stderr, [ + 'Error running tsd:', + 'JSONError: Unexpected end of JSON input while parsing empty string', + 'at parseJson (/Users/tommymitchell/src/_open-source/tsd/node_modules/parse-json/index.js:29:21)', + 'at module.exports (/Users/tommymitchell/src/_open-source/tsd/node_modules/read-pkg/index.js:17:15)', + 'at async module.exports (/Users/tommymitchell/src/_open-source/tsd/node_modules/read-pkg-up/index.js:14:16)', + ], {startLine: 0}); }); test('exported formatter matches cli results', async t => { @@ -114,7 +145,11 @@ test('exported formatter matches cli results', async t => { const {stderr: cliResults} = await t.throwsAsync(execa('../../../cli.js', options)); - t.true(cliResults.includes('✖ 5:19 Argument of type number is not assignable to parameter of type string.')); + verifyCli(t, cliResults, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); const tsdResults = await tsd(options); const formattedResults = formatter(tsdResults); diff --git a/source/test/diff.ts b/source/test/diff.ts index e2a26a1a..b8867618 100644 --- a/source/test/diff.ts +++ b/source/test/diff.ts @@ -1,4 +1,4 @@ -import {verifyWithDiff} from './fixtures/utils'; +import {verifyWithDiff, verifyCli} from './fixtures/utils'; import execa, {ExecaError} from 'execa'; import path from 'path'; import test from 'ava'; @@ -41,8 +41,7 @@ test('diff cli', async t => { const {exitCode, stderr} = await t.throwsAsync(execa('dist/cli.js', [file, '--show-diff'])); t.is(exitCode, 1); - - const expectedLines = [ + verifyCli(t, stderr, [ '✖ 8:0 Parameter type { life?: number | undefined; } is declared too wide for argument type { life: number; }.', '', '- { life?: number | undefined; }', @@ -69,13 +68,5 @@ test('diff cli', async t => { '+ This is a comment.', '', '6 errors' - ]; - - // NOTE: If lines are added to the output in the future startLine and endLine should be adjusted. - const startLine = 2; // Skip tsd error message and file location. - const endLine = startLine + expectedLines.length; // Grab diff output only and skip stack trace. - - const receivedLines = stderr.trim().split('\n').slice(startLine, endLine).map(line => line.trim()); - - t.deepEqual(receivedLines, expectedLines); + ]); }); diff --git a/source/test/fixtures/utils.ts b/source/test/fixtures/utils.ts index 4cbea202..0144539b 100644 --- a/source/test/fixtures/utils.ts +++ b/source/test/fixtures/utils.ts @@ -94,7 +94,7 @@ export const verifyWithFileName = ( * @param diagnostics - List of diagnostics to verify. * @param expectations - Expected diagnostics. */ - export const verifyWithDiff = ( +export const verifyWithDiff = ( t: ExecutionContext, diagnostics: Diagnostic[], expectations: ExpectationWithDiff[] @@ -117,3 +117,25 @@ export const verifyWithFileName = ( t.deepEqual(diagnosticObjs, expectationObjs, 'Received diagnostics that are different from expectations!'); }; + +/** + * Verify a list of diagnostics reported from the CLI. + * + * @param t - The AVA execution context. + * @param diagnostics - List of diagnostics to verify. + * @param expectations - Expected diagnostics. + * @param startLine - Optionally specify how many lines to skip from start. + */ +export const verifyCli = ( + t: ExecutionContext, + diagnostics: string, + expectedLines: string[], + {startLine}: {startLine: number} = {startLine: 1} // Skip file location. +) => { + // NOTE: If lines are added to the output in the future `startLine` and `endLine` should be adjusted. + const endLine = startLine + expectedLines.length; // Grab diff output only and skip stack trace. + + const receivedLines = diagnostics.trim().split('\n').slice(startLine, endLine).map(line => line.trim()); + + t.deepEqual(receivedLines, expectedLines, 'Received diagnostics that are different from expectations!'); +}; From dafe675d824e163e0be8e7f2933be23417de22cd Mon Sep 17 00:00:00 2001 From: Tommy Date: Tue, 14 Mar 2023 00:42:58 -0500 Subject: [PATCH 4/8] =?UTF-8?q?fix(`test/cli.ts`):=20don=E2=80=99t=20test?= =?UTF-8?q?=20exact=20path=20in=20error=20stack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- source/test/cli.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/test/cli.ts b/source/test/cli.ts index 891fbe25..366501f8 100644 --- a/source/test/cli.ts +++ b/source/test/cli.ts @@ -132,9 +132,8 @@ test('tsd logs stacktrace on failure', async t => { verifyCli(t, stderr, [ 'Error running tsd:', 'JSONError: Unexpected end of JSON input while parsing empty string', - 'at parseJson (/Users/tommymitchell/src/_open-source/tsd/node_modules/parse-json/index.js:29:21)', - 'at module.exports (/Users/tommymitchell/src/_open-source/tsd/node_modules/read-pkg/index.js:17:15)', - 'at async module.exports (/Users/tommymitchell/src/_open-source/tsd/node_modules/read-pkg-up/index.js:14:16)', + // TODO: check that stack matches without checking for exact filepath + // would have to match in CI and locally ], {startLine: 0}); }); From c7ec3a316ab06d8be52e06699600ca23e61b68c8 Mon Sep 17 00:00:00 2001 From: tommy-mitchell Date: Tue, 14 Mar 2023 01:21:39 -0500 Subject: [PATCH 5/8] fix: cli success detection on Node 14 --- source/cli.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/cli.ts b/source/cli.ts index 7d8e0215..5951cc4a 100644 --- a/source/cli.ts +++ b/source/cli.ts @@ -45,6 +45,7 @@ const cli = meow(` }); (async () => { + let success = false; try { const cwd = cli.input.length > 0 ? cli.input[0] : process.cwd(); const {typings: typingsFile, files: testFiles, showDiff} = cli.flags; @@ -54,15 +55,14 @@ const cli = meow(` const diagnostics = await tsd(options); if (diagnostics.length > 0) { - throw new Error(formatter(diagnostics, showDiff), {cause: 'tsd found diagnostics'}); + success = true; + throw new Error(formatter(diagnostics, showDiff)); } } catch (error: unknown) { const potentialError = error as Error | undefined; - if (potentialError?.cause === 'tsd found diagnostics') { - if (potentialError?.message) { - console.error(potentialError?.message); - } + if (success && potentialError?.message) { + console.error(potentialError?.message); } else if (potentialError?.stack) { console.error(`Error running tsd:\n${potentialError?.stack}`); } From ee0bc039f85b9019e09d8762d3a00f24eb7db0d7 Mon Sep 17 00:00:00 2001 From: tommy-mitchell Date: Tue, 14 Mar 2023 11:56:17 -0500 Subject: [PATCH 6/8] fix: only throw on failure or if tsd finds errors --- source/cli.ts | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/source/cli.ts b/source/cli.ts index 5951cc4a..89ac8e6c 100644 --- a/source/cli.ts +++ b/source/cli.ts @@ -44,29 +44,39 @@ const cli = meow(` }, }); +/** + * Displays a message and exits, conditionally erroring. + * + * @param message The message to display. + * @param isError Whether or not to fail on exit. + */ +const exit = (message: string, {isError = true}: {isError?: boolean} = {}) => { + if (isError) { + console.error(message); + process.exit(1); + } else { + console.log(message); + process.exit(0); + } +}; + (async () => { - let success = false; try { const cwd = cli.input.length > 0 ? cli.input[0] : process.cwd(); const {typings: typingsFile, files: testFiles, showDiff} = cli.flags; - const options = {cwd, typingsFile, testFiles}; - - const diagnostics = await tsd(options); + const diagnostics = await tsd({cwd, typingsFile, testFiles}); if (diagnostics.length > 0) { - success = true; - throw new Error(formatter(diagnostics, showDiff)); + const hasErrors = diagnostics.some(diagnostic => diagnostic.severity === 'error'); + const formattedDiagnostics = formatter(diagnostics, showDiff); + + exit(formattedDiagnostics, {isError: hasErrors}); } } catch (error: unknown) { const potentialError = error as Error | undefined; + const errorMessage = potentialError?.stack ?? potentialError?.message ?? 'tsd unexpectedly crashed.'; - if (success && potentialError?.message) { - console.error(potentialError?.message); - } else if (potentialError?.stack) { - console.error(`Error running tsd:\n${potentialError?.stack}`); - } - - process.exit(1); + exit(`Error running tsd:\n${errorMessage}`); } })(); From 9e6a4ee580710571e6d68e90712917254c9dabce Mon Sep 17 00:00:00 2001 From: tommy-mitchell Date: Tue, 14 Mar 2023 12:40:23 -0500 Subject: [PATCH 7/8] fix(`verifyCli`): exact matching --- source/test/cli.ts | 14 ++++++++++---- source/test/fixtures/utils.ts | 5 +---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/source/test/cli.ts b/source/test/cli.ts index 366501f8..441a0016 100644 --- a/source/test/cli.ts +++ b/source/test/cli.ts @@ -127,13 +127,15 @@ test('tsd logs stacktrace on failure', async t => { cwd: path.join(__dirname, 'fixtures/empty-package-json') })); - t.is(exitCode, 1); + const nodeModulesPath = path.resolve('node_modules'); + t.is(exitCode, 1); verifyCli(t, stderr, [ 'Error running tsd:', 'JSONError: Unexpected end of JSON input while parsing empty string', - // TODO: check that stack matches without checking for exact filepath - // would have to match in CI and locally + `at parseJson (${nodeModulesPath}/parse-json/index.js:29:21)`, + `at module.exports (${nodeModulesPath}/read-pkg/index.js:17:15)`, + `at async module.exports (${nodeModulesPath}/read-pkg-up/index.js:14:16)`, ], {startLine: 0}); }); @@ -153,5 +155,9 @@ test('exported formatter matches cli results', async t => { const tsdResults = await tsd(options); const formattedResults = formatter(tsdResults); - t.true(formattedResults.includes('✖ 5:19 Argument of type number is not assignable to parameter of type string.')); + verifyCli(t, formattedResults, [ + '✖ 5:19 Argument of type number is not assignable to parameter of type string.', + '', + '1 error', + ]); }); diff --git a/source/test/fixtures/utils.ts b/source/test/fixtures/utils.ts index 0144539b..a65fdfd5 100644 --- a/source/test/fixtures/utils.ts +++ b/source/test/fixtures/utils.ts @@ -132,10 +132,7 @@ export const verifyCli = ( expectedLines: string[], {startLine}: {startLine: number} = {startLine: 1} // Skip file location. ) => { - // NOTE: If lines are added to the output in the future `startLine` and `endLine` should be adjusted. - const endLine = startLine + expectedLines.length; // Grab diff output only and skip stack trace. - - const receivedLines = diagnostics.trim().split('\n').slice(startLine, endLine).map(line => line.trim()); + const receivedLines = diagnostics.trim().split('\n').slice(startLine).map(line => line.trim()); t.deepEqual(receivedLines, expectedLines, 'Received diagnostics that are different from expectations!'); }; From 1a9c8098f9926ae01c39057c07fcf034c83422b4 Mon Sep 17 00:00:00 2001 From: tommy-mitchell Date: Tue, 14 Mar 2023 13:13:58 -0500 Subject: [PATCH 8/8] fix: stacktrace resolution for ci --- package.json | 1 + source/test/cli.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 1bc87b81..52b9084a 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "eslint-config-xo-typescript": "^0.41.1", "execa": "^5.0.0", "react": "^16.9.0", + "resolve-from": "^5.0.0", "rxjs": "^6.5.3", "typescript": "~4.9.5" }, diff --git a/source/test/cli.ts b/source/test/cli.ts index 441a0016..1f5d510a 100644 --- a/source/test/cli.ts +++ b/source/test/cli.ts @@ -4,6 +4,7 @@ import execa from 'execa'; import readPkgUp from 'read-pkg-up'; import tsd, {formatter} from '..'; import {verifyCli} from './fixtures/utils'; +import resolveFrom from 'resolve-from'; interface ExecaError extends Error { readonly exitCode: number; @@ -128,12 +129,13 @@ test('tsd logs stacktrace on failure', async t => { })); const nodeModulesPath = path.resolve('node_modules'); + const parseJsonPath = resolveFrom.silent(`${nodeModulesPath}/read-pkg`, 'parse-json') ?? `${nodeModulesPath}/index.js`; t.is(exitCode, 1); verifyCli(t, stderr, [ 'Error running tsd:', 'JSONError: Unexpected end of JSON input while parsing empty string', - `at parseJson (${nodeModulesPath}/parse-json/index.js:29:21)`, + `at parseJson (${parseJsonPath}:29:21)`, `at module.exports (${nodeModulesPath}/read-pkg/index.js:17:15)`, `at async module.exports (${nodeModulesPath}/read-pkg-up/index.js:14:16)`, ], {startLine: 0});