feat(web-scripts preinstall): Add test coverage and use enum options

For test coverage, I added multiple package.json files that demonstrated what various
vulnerabilities and threshold limits should show a user.

re #73

package.json

@@ -24,6 +24,9 @@
     "typescript": "^3.4.3"
   },
   "resolutions": {
+    "**/**/handlebars": "4.5.3",
+    "**/**/marked": "0.8.0",
+    "**/**/https-proxy-agent": "2.2.3",
     "lodash": ">=4.17.12",
     "mixin-deep": ">=1.3.2",
     "set-value": ">=3.0.1"

packages/web-scripts/src/SharedTypes.ts

@@ -1,3 +1,11 @@
+type ThresholdLimits =
+  | 'info'
+  | 'low'
+  | 'moderate'
+  | 'high'
+  | 'critical'
+  | 'none';
+
 export type TaskName =
   | 'init'
   | 'build'
@@ -54,7 +62,7 @@ export type ReleaseTaskDesc = {
 
 export type PreinstallTaskDesc = {
   name: 'preinstall';
-  threshold?: number;
+  threshold: ThresholdLimits;
 } & TaskDesc;
 
 export type PrecommitTaskDesc = {

packages/web-scripts/src/Tasks/Preinstall.test.ts

@@ -0,0 +1,52 @@
+import { preinstallTask } from './PreinstallTasks';
+import { THIS_ROOT } from '../Paths';
+
+// @ts-ignore
+jest.spyOn(process, 'exit').mockImplementation(c => c);
+
+/**
+ * WARNING:
+ * These tests have some issues with being potentially non-deterministic.
+ * Due to a network dependency on npmjs.com this test could potentially fail
+ * should there be any downtime with external services used by with yarn audit.
+ *
+ * Should these tests begin to fail suddenly, it might be worth trading test coverage
+ * confidence for test reliability by mocking the network calls made by yarn audit.
+ */
+describe('web-scripts preinstall', () => {
+  beforeEach(() => jest.clearAllMocks());
+  afterAll(() => jest.restoreAllMocks());
+
+  test.each`
+    violations | threshold     | status
+    ${0}       | ${undefined}  | ${0}
+    ${0}       | ${'none'}     | ${0}
+    ${12}      | ${undefined}  | ${0}
+    ${12}      | ${'none'}     | ${0}
+    ${12}      | ${'low'}      | ${12}
+    ${12}      | ${'moderate'} | ${12}
+    ${12}      | ${'high'}     | ${12}
+    ${12}      | ${'critical'} | ${0}
+    ${30}      | ${undefined}  | ${0}
+    ${30}      | ${'none'}     | ${0}
+    ${30}      | ${'low'}      | ${30}
+    ${30}      | ${'moderate'} | ${30}
+    ${30}      | ${'high'}     | ${30}
+    ${30}      | ${'critical'} | ${30}
+  `(
+    'return status code $status when audited dependencies have $violations violations and $threshold threshold',
+    async ({ violations, threshold, status }) => {
+      const source = `${THIS_ROOT}/src/Tasks/__fixtures__/preinstall/${violations}`;
+
+      await preinstallTask({
+        name: 'preinstall',
+        threshold,
+        // Overrides implementation logic for CWD
+        restOptions: ['--cwd', source],
+      });
+
+      if (status) expect(process.exit).toHaveBeenCalledWith(status);
+      else expect(process.exit).not.toHaveBeenCalled();
+    },
+  );
+});

packages/web-scripts/src/Tasks/PreinstallTasks.ts

@@ -6,6 +6,15 @@ import { CONSUMING_ROOT } from '../Paths';
 
 const dbg = Debug('web-scripts:preinstall'); // eslint-disable-line new-cap
 
+enum ThresholdLimits {
+  info = 1,
+  low = 2,
+  moderate = 4,
+  high = 8,
+  critical = 16,
+  none = 32,
+}
+
 export async function preinstallTask(
   task: PreinstallTaskDesc,
 ): Promise<string[]> {
@@ -36,7 +45,7 @@ export async function preinstallTask(
  */
 async function yarnRun(task: PreinstallTaskDesc): Promise<string> {
   const cmd = 'npx';
-  const { threshold = 32 } = task;
+  const { threshold } = task;
 
   const args = [
     '--no-install',
@@ -49,11 +58,11 @@ async function yarnRun(task: PreinstallTaskDesc): Promise<string> {
   dbg('npx args %o', args);
 
   try {
-    const stdout = await spawn(cmd, args, { stdio: 'inherit' });
-    return (stdout || '').toString();
+    await spawn(cmd, args, { stdio: 'inherit' });
   } catch (err) {
-    const thresholdReached = err.exitStatus >= threshold;
+    const thresholdReached = err.exitStatus >= ThresholdLimits[threshold];
     if (thresholdReached) process.exit(err.exitStatus);
-    return '';
   }
+
+  return '';
 }

packages/web-scripts/src/Tasks/__fixtures__/preinstall/0/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "preinstall-test",
+  "description": "This test has no dependencies, so the preinstall dependency should return a status code of 0"
+}

packages/web-scripts/src/Tasks/__fixtures__/preinstall/12/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "preinstall-test",
+  "description": "Throws a high and moderate status. See => https://www.npmjs.com/advisories/10",
+  "devDependencies": {
+    "geddy": "13.0.7"
+  }
+}

packages/web-scripts/src/Tasks/__fixtures__/preinstall/30/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "preinstall-test",
+  "description": "Throws a high and moderate status. See => https://www.npmjs.com/advisories/10 + 48 + 1 ",
+  "devDependencies": {
+    "bassmaster": "1.5.1",
+    "geddy": "13.0.7",
+    "uglify-js": "2.5.0",
+    "chokidar": "2.1.8"
+  }
+}

packages/web-scripts/src/index.ts

@@ -143,13 +143,31 @@ program
     handleSpawnResult(precommitTask(t));
   });
 
+function thresholdLimiter(value: string, defaultValue: string) {
+  switch (value) {
+    case 'info':
+    case 'low':
+    case 'moderate':
+    case 'high':
+    case 'critical':
+      return value;
+    default:
+      return defaultValue;
+  }
+}
+
 program
   .command('preinstall')
   .allowUnknownOption()
   .description(
     `Run yarn audit and exit non-zero if the security vulnerability threshold is breached`,
   )
-  .option('--threshold [level]', 'The amount of permissible vulnerabilities')
+  .option(
+    '--threshold [info|low|moderate|high|critical]',
+    'The amount of permissible vulnerabilities',
+    thresholdLimiter,
+    'none',
+  )
   .action((...args) => {
     const cmd = getCommand(args);
     const rest = getPositionalArgs(args);