From aaed1878184f9717fcdef34c9be1a2a7245b2d5f Mon Sep 17 00:00:00 2001 From: Joseph Casey Date: Sat, 28 Dec 2019 16:13:01 -0500 Subject: [PATCH] feat(web-scripts preinstall): Add test coverage and use enum options For test coverage, I added multiple package.json files that demonstrated what various vulnerabilities and threshold limits should show a user. re #73 --- package.json | 3 ++ packages/web-scripts/src/SharedTypes.ts | 10 +++- .../web-scripts/src/Tasks/Preinstall.test.ts | 52 +++++++++++++++++++ .../web-scripts/src/Tasks/PreinstallTasks.ts | 19 +++++-- .../__fixtures__/preinstall/0/package.json | 4 ++ .../__fixtures__/preinstall/12/package.json | 7 +++ .../__fixtures__/preinstall/30/package.json | 10 ++++ packages/web-scripts/src/index.ts | 20 ++++++- 8 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 packages/web-scripts/src/Tasks/Preinstall.test.ts create mode 100644 packages/web-scripts/src/Tasks/__fixtures__/preinstall/0/package.json create mode 100644 packages/web-scripts/src/Tasks/__fixtures__/preinstall/12/package.json create mode 100644 packages/web-scripts/src/Tasks/__fixtures__/preinstall/30/package.json diff --git a/package.json b/package.json index d343f86e..daca7313 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,9 @@ "typescript": "^3.4.3" }, "resolutions": { + "**/**/handlebars": "4.5.3", + "**/**/marked": "0.8.0", + "**/**/https-proxy-agent": "2.2.3", "lodash": ">=4.17.12", "mixin-deep": ">=1.3.2", "set-value": ">=3.0.1" diff --git a/packages/web-scripts/src/SharedTypes.ts b/packages/web-scripts/src/SharedTypes.ts index 28998607..88cd8744 100644 --- a/packages/web-scripts/src/SharedTypes.ts +++ b/packages/web-scripts/src/SharedTypes.ts @@ -1,3 +1,11 @@ +type ThresholdLimits = + | 'info' + | 'low' + | 'moderate' + | 'high' + | 'critical' + | 'none'; + export type TaskName = | 'init' | 'build' @@ -54,7 +62,7 @@ export type ReleaseTaskDesc = { export type PreinstallTaskDesc = { name: 'preinstall'; - threshold?: number; + threshold: ThresholdLimits; } & TaskDesc; export type PrecommitTaskDesc = { diff --git a/packages/web-scripts/src/Tasks/Preinstall.test.ts b/packages/web-scripts/src/Tasks/Preinstall.test.ts new file mode 100644 index 00000000..632ba907 --- /dev/null +++ b/packages/web-scripts/src/Tasks/Preinstall.test.ts @@ -0,0 +1,52 @@ +import { preinstallTask } from './PreinstallTasks'; +import { THIS_ROOT } from '../Paths'; + +// @ts-ignore +jest.spyOn(process, 'exit').mockImplementation(c => c); + +/** + * WARNING: + * These tests have some issues with being potentially non-deterministic. + * Due to a network dependency on npmjs.com this test could potentially fail + * should there be any downtime with external services used by with yarn audit. + * + * Should these tests begin to fail suddenly, it might be worth trading test coverage + * confidence for test reliability by mocking the network calls made by yarn audit. + */ +describe('web-scripts preinstall', () => { + beforeEach(() => jest.clearAllMocks()); + afterAll(() => jest.restoreAllMocks()); + + test.each` + violations | threshold | status + ${0} | ${undefined} | ${0} + ${0} | ${'none'} | ${0} + ${12} | ${undefined} | ${0} + ${12} | ${'none'} | ${0} + ${12} | ${'low'} | ${12} + ${12} | ${'moderate'} | ${12} + ${12} | ${'high'} | ${12} + ${12} | ${'critical'} | ${0} + ${30} | ${undefined} | ${0} + ${30} | ${'none'} | ${0} + ${30} | ${'low'} | ${30} + ${30} | ${'moderate'} | ${30} + ${30} | ${'high'} | ${30} + ${30} | ${'critical'} | ${30} + `( + 'return status code $status when audited dependencies have $violations violations and $threshold threshold', + async ({ violations, threshold, status }) => { + const source = `${THIS_ROOT}/src/Tasks/__fixtures__/preinstall/${violations}`; + + await preinstallTask({ + name: 'preinstall', + threshold, + // Overrides implementation logic for CWD + restOptions: ['--cwd', source], + }); + + if (status) expect(process.exit).toHaveBeenCalledWith(status); + else expect(process.exit).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/packages/web-scripts/src/Tasks/PreinstallTasks.ts b/packages/web-scripts/src/Tasks/PreinstallTasks.ts index cb3ba98a..328b4ca1 100644 --- a/packages/web-scripts/src/Tasks/PreinstallTasks.ts +++ b/packages/web-scripts/src/Tasks/PreinstallTasks.ts @@ -6,6 +6,15 @@ import { CONSUMING_ROOT } from '../Paths'; const dbg = Debug('web-scripts:preinstall'); // eslint-disable-line new-cap +enum ThresholdLimits { + info = 1, + low = 2, + moderate = 4, + high = 8, + critical = 16, + none = 32, +} + export async function preinstallTask( task: PreinstallTaskDesc, ): Promise { @@ -36,7 +45,7 @@ export async function preinstallTask( */ async function yarnRun(task: PreinstallTaskDesc): Promise { const cmd = 'npx'; - const { threshold = 32 } = task; + const { threshold } = task; const args = [ '--no-install', @@ -49,11 +58,11 @@ async function yarnRun(task: PreinstallTaskDesc): Promise { dbg('npx args %o', args); try { - const stdout = await spawn(cmd, args, { stdio: 'inherit' }); - return (stdout || '').toString(); + await spawn(cmd, args, { stdio: 'inherit' }); } catch (err) { - const thresholdReached = err.exitStatus >= threshold; + const thresholdReached = err.exitStatus >= ThresholdLimits[threshold]; if (thresholdReached) process.exit(err.exitStatus); - return ''; } + + return ''; } diff --git a/packages/web-scripts/src/Tasks/__fixtures__/preinstall/0/package.json b/packages/web-scripts/src/Tasks/__fixtures__/preinstall/0/package.json new file mode 100644 index 00000000..c9307945 --- /dev/null +++ b/packages/web-scripts/src/Tasks/__fixtures__/preinstall/0/package.json @@ -0,0 +1,4 @@ +{ + "name": "preinstall-test", + "description": "This test has no dependencies, so the preinstall dependency should return a status code of 0" +} diff --git a/packages/web-scripts/src/Tasks/__fixtures__/preinstall/12/package.json b/packages/web-scripts/src/Tasks/__fixtures__/preinstall/12/package.json new file mode 100644 index 00000000..b8a43188 --- /dev/null +++ b/packages/web-scripts/src/Tasks/__fixtures__/preinstall/12/package.json @@ -0,0 +1,7 @@ +{ + "name": "preinstall-test", + "description": "Throws a high and moderate status. See => https://www.npmjs.com/advisories/10", + "devDependencies": { + "geddy": "13.0.7" + } +} diff --git a/packages/web-scripts/src/Tasks/__fixtures__/preinstall/30/package.json b/packages/web-scripts/src/Tasks/__fixtures__/preinstall/30/package.json new file mode 100644 index 00000000..81fd9ea3 --- /dev/null +++ b/packages/web-scripts/src/Tasks/__fixtures__/preinstall/30/package.json @@ -0,0 +1,10 @@ +{ + "name": "preinstall-test", + "description": "Throws a high and moderate status. See => https://www.npmjs.com/advisories/10 + 48 + 1 ", + "devDependencies": { + "bassmaster": "1.5.1", + "geddy": "13.0.7", + "uglify-js": "2.5.0", + "chokidar": "2.1.8" + } +} diff --git a/packages/web-scripts/src/index.ts b/packages/web-scripts/src/index.ts index 4a22a09b..3afffc93 100644 --- a/packages/web-scripts/src/index.ts +++ b/packages/web-scripts/src/index.ts @@ -143,13 +143,31 @@ program handleSpawnResult(precommitTask(t)); }); +function thresholdLimiter(value: string, defaultValue: string) { + switch (value) { + case 'info': + case 'low': + case 'moderate': + case 'high': + case 'critical': + return value; + default: + return defaultValue; + } +} + program .command('preinstall') .allowUnknownOption() .description( `Run yarn audit and exit non-zero if the security vulnerability threshold is breached`, ) - .option('--threshold [level]', 'The amount of permissible vulnerabilities') + .option( + '--threshold [info|low|moderate|high|critical]', + 'The amount of permissible vulnerabilities', + thresholdLimiter, + 'none', + ) .action((...args) => { const cmd = getCommand(args); const rest = getPositionalArgs(args);