feat: added configurable warning limits through .thresholdrc file

- added code to process a .thresholdrc file to get warnings limit
- used warning limit to set exit code to 1 and print a message if warning limit is exceeded
- added validation on the user input from .thresholdrc to check if valid keys and values provided in .thresholdrc
- set default limit to Number.MAX_VALUE if no .thresholdrc provided
- added tests to test error handling for .thresholdrc file and expected output when warning limits exceeded

src/cli-validator/runValidator.js

@@ -41,6 +41,8 @@ const processInput = async function(program) {
 
   const configFileOverride = program.config;
 
+  const limitsFileOverride = program.limits;
+
   // turn on coloring by default
   const colors = turnOffColoring ? false : true;
 
@@ -140,6 +142,14 @@ const processInput = async function(program) {
     return Promise.reject(err);
   }
 
+  // get limits from .thresholdrc file
+  let limitsObject;
+  try {
+    limitsObject = await config.limits(chalk, limitsFileOverride);
+  } catch (err) {
+    return Promise.reject(err);
+  }
+
   // define an exit code to return. this will tell the parent program whether
   // the validator passed or not
   let exitCode = 0;
@@ -250,8 +260,22 @@ const processInput = async function(program) {
           originalFile,
           errorsOnly
         );
-        // fail on errors, but not if there are only warnings
-        if (results.error) exitCode = 1;
+        // fail on errors or if number of warnings exceeds warnings limit
+        if (results.error) {
+          exitCode = 1;
+        } else {
+          // Calculate number of warnings and set exit code to 1 if warning limit exceeded
+          let numWarnings = 0;
+          for (const key of Object.keys(results.warnings)) {
+            numWarnings += results.warnings[key].length;
+          }
+          if (numWarnings > limitsObject.warnings) {
+            exitCode = 1;
+            console.log(
+              chalk.red(`Number of warnings (${numWarnings}) exceeds warnings limit (${limitsObject.warnings}).`)
+            );
+          }
+        }
       } else {
         console.log(chalk.green(`\n${validFile} passed the validator`));
         if (validFile === last(filesToValidate)) console.log();

src/cli-validator/utils/processConfiguration.js

@@ -13,6 +13,26 @@ const defaultObject = defaultConfig.defaults;
 const deprecatedRuleObject = defaultConfig.deprecated;
 const configOptions = defaultConfig.options;
 
+printConfigErrors = function(problems, chalk, fileName) {
+  const description = `Invalid configuration in ${chalk.underline(
+    fileName
+  )} file. See below for details.`;
+
+  const message = [];
+
+  // add all errors for printError
+  problems.forEach(function(problem) {
+    message.push(
+      `\n - ${chalk.red(problem.message)}\n   ${chalk.magenta(
+        problem.correction
+      )}`
+    );
+  });
+  if (message.length) {
+    printError(chalk, description, message.join('\n'));
+  }
+}
+
 const validateConfigObject = function(configObject, chalk) {
   const configErrors = [];
   let validObject = true;
@@ -147,21 +167,8 @@ const validateConfigObject = function(configObject, chalk) {
     configObject.invalid = false;
   } else {
     // if the object is not valid, exit and tell the user why
-    const description = `Invalid configuration in ${chalk.underline(
-      '.validaterc'
-    )} file. See below for details.`;
-    const message = [];
-
-    // concatenate all the error messages for the printError module
-    configErrors.forEach(function(problem) {
-      message.push(
-        `\n - ${chalk.red(problem.message)}\n   ${chalk.magenta(
-          problem.correction
-        )}`
-      );
-    });
+    printConfigErrors(configErrors, chalk, '.validaterc');
 
-    printError(chalk, description, message.join('\n'));
     configObject.invalid = true;
   }
 
@@ -263,6 +270,86 @@ const getFilesToIgnore = async function() {
   return filesToIgnore;
 };
 
+const validateLimits = function(limitsObject, chalk) {
+  const allowedLimits = ['warnings'];
+  const limitErrors = [];
+
+  Object.keys(limitsObject).forEach(function(key) {
+    if (!allowedLimits.includes(key)) {
+      // remove the entry and notify the user
+      delete limitsObject[key];
+      limitErrors.push({
+        message: `"${key}" limit not supported. This value will be ignored.`,
+        correction: `Valid limits for .thresholdrc are: ${allowedLimits.join(', ')}.`
+      });
+    } else {
+      // valid limit option, ensure the limit given is a number
+      if (typeof limitsObject[key] !== 'number') {
+          // remove the entry and notify the user
+          delete limitsObject[key];
+          limitErrors.push({
+            message: `Value provided for ${key} limit is invalid.`,
+            correction: `${key} limit should be a number.`
+          });
+      }
+    }
+  });
+
+  // give the user corrections for .thresholdrc file
+  if (limitErrors.length) {
+    console.log("there are errors");
+    printConfigErrors(limitErrors, chalk, '.thresholdrc');
+  }
+
+  //  sets all limits options not defined by user to default
+  for (let limitOption of allowedLimits) {
+    if (!(limitOption in limitsObject)) {
+      limitsObject[limitOption] = Number.MAX_VALUE;
+    }
+  }
+
+  return limitsObject;
+}
+
+const getLimits = async function(chalk, limitsFileOverride) {
+  let limitsObject = {};
+
+  const findUpOpts = {};
+  let limitsFileName;
+
+  if (limitsFileOverride) {
+    limitsFileName = path.basename(limitsFileOverride);
+    findUpOpts.cwd = path.dirname(limitsFileOverride);
+  } else {
+    limitsFileName = '.thresholdrc';
+  }
+
+  // search up the file system for the first instance
+  // of the threshold file
+  limitsFile = await findUp(limitsFileName, findUpOpts);
+
+  if (limitsFile !== null) {
+    try {
+      // the threshold file must be in the root folder of the project
+      const fileAsString = await readFile(limitsFile, 'utf8');
+      limitsObject = JSON.parse(fileAsString);
+
+    } catch (err) {
+      // this most likely means there is a problem in the json syntax itself
+      const description =
+        'There is a problem with the .thresholdrc file. See below for details.';
+      printError(chalk, description, err);
+      return Promise.reject(2);
+    }
+  }
+
+  // returns complete limits object with all valid user settings
+  // and default values for undefined limits
+  limitsObject = validateLimits(limitsObject, chalk);
+
+  return limitsObject;
+};
+
 const validateConfigOption = function(userOption, defaultOption) {
   const result = { valid: true };
   // determine what type of option it is
@@ -286,3 +373,5 @@ module.exports.get = getConfigObject;
 module.exports.validate = validateConfigObject;
 module.exports.ignore = getFilesToIgnore;
 module.exports.validateOption = validateConfigOption;
+module.exports.validateLimits = validateLimits;
+module.exports.limits = getLimits;

test/cli-validator/mockFiles/thresholds/.fiveWarnings

@@ -0,0 +1,3 @@
+{
+    "warnings": 5
+}

test/cli-validator/mockFiles/thresholds/.invalidValues

@@ -0,0 +1,5 @@
+{
+    "errors": 0,
+    "warnings": "text",
+    "population": 10
+}

test/cli-validator/mockFiles/thresholds/.zeroWarnings

@@ -0,0 +1,3 @@
+{
+    "warnings": 0
+}

test/cli-validator/tests/thresholdValidator.js

@@ -0,0 +1,82 @@
+// the rule names are all snake case and need to stay that way. don't lint them
+/* eslint-disable camelcase */
+
+const intercept = require('intercept-stdout');
+const expect = require('expect');
+const stripAnsiFrom = require('strip-ansi');
+const chalk = require('chalk');
+
+const commandLineValidator = require('../../../src/cli-validator/runValidator');
+
+describe('test the .thresholdrc limits', function() {
+  it('should show error and set exit code to 1 when warning limit exceeded', async function() {
+
+    const capturedText = [];
+
+    const program = {};
+    program.args = ['./test/cli-validator/mockFiles/circularRefs.yml'];
+    program.limits = 
+      './test/cli-validator/mockFiles/thresholds/.fiveWarnings';
+    program.default_mode = true;
+
+    const unhookIntercept = intercept(function(txt) {
+      capturedText.push(stripAnsiFrom(txt));
+      return '';
+    });
+
+    const exitCode = await commandLineValidator(program);
+
+    unhookIntercept();
+
+    expect(exitCode).toEqual(1);
+
+    expect(capturedText[capturedText.length - 1].slice(0, 18)).toEqual(
+      `Number of warnings`
+    );
+  });
+
+  it('should print errors for unsupported limit options and invalid limit values', async function() {
+
+    const capturedText = [];
+
+    const program = {};
+    program.args = ['./test/cli-validator/mockFiles/clean.yml'];
+    program.limits = 
+      './test/cli-validator/mockFiles/thresholds/.invalidValues';
+    program.default_mode = true;
+
+    const unhookIntercept = intercept(function(txt) {
+      capturedText.push(stripAnsiFrom(txt));
+      return '';
+    });
+
+    const exitCode = await commandLineValidator(program);
+
+    unhookIntercept();
+
+    // limit values invalid, so default limit, Number.MAX_VALUE, used
+    expect(exitCode).toEqual(0);
+
+    const allOutput = capturedText.join('');
+
+    expect(
+      allOutput.includes('\"population\" limit not supported.') &&
+      allOutput.includes('Value provided for warnings')
+    ).toEqual(true);
+
+  });
+
+  it('should give exit code 0 when warnings limit not exceeded', async function() {
+
+    const program = {};
+    program.args = ['./test/cli-validator/mockFiles/clean.yml'];
+    program.limits = 
+      './test/cli-validator/mockFiles/thresholds/.zeroWarnings';
+    program.default_mode = true;
+
+    const exitCode = await commandLineValidator(program);
+
+    expect(exitCode).toEqual(0);
+  });
+
+});