New: --max-fatal-errors option (eslint#13711)

docs/user-guide/command-line-interface.md

@@ -61,6 +61,7 @@ Using stdin:
 Handling warnings:
   --quiet                        Report errors only - default: false
   --max-warnings Int             Number of warnings to trigger nonzero exit code - default: -1
+  --max-fatal-errors Int         Number of fatal errors to trigger exit code 3 - default: -1
 
 Output:
   -o, --output-file path::String  Specify file to write report to
@@ -319,6 +320,26 @@ Example:
 
     eslint --max-warnings 10 file.js
 
+#### `--max-fatal-errors`
+
+This option allows you to specify a fatal error threshold, which can be used to force ESLint to exit with an error code 3 if there are too many errors in your
+project that are considered fatal.
+
+Normally, ESLint treats fatal and non-fatal errors in the same way and exits with error code 1 if there is at least one. Error code 1 code indicates that linting
+was successful but fatal errors are often parsing errors that effectively prevented the linting. Many are caused by misconfigurations and fixable. This threshold
+enables ESLint to explicitly warn user with error message and different exit code.
+
+Example:
+
+    eslint --max-fatal-errors 0 file.js
+
+    file.js
+      1:1  error  Parsing error: The keyword 'import' is reserved
+
+    ✖ 1 problem (1 error, 0 warnings)
+
+    ESLint found too many fatal parsing errors (maximum: 0).
+
 ### Output
 
 #### `-o`, `--output-file`

lib/cli-engine/cli-engine.js

@@ -94,6 +94,7 @@ const validFixTypes = new Set(["problem", "suggestion", "layout"]);
  * @typedef {Object} LintReport
  * @property {LintResult[]} results All of the result.
  * @property {number} errorCount Number of errors for the result.
+ * @property {number} fatalErrorCount Number of fatal errors for the result.
  * @property {number} warningCount Number of warnings for the result.
  * @property {number} fixableErrorCount Number of fixable errors for the result.
  * @property {number} fixableWarningCount Number of fixable warnings for the result.
@@ -146,6 +147,9 @@ function calculateStatsPerFile(messages) {
     return messages.reduce((stat, message) => {
         if (message.fatal || message.severity === 2) {
             stat.errorCount++;
+            if (message.fatal) {
+                stat.fatalErrorCount++;
+            }
             if (message.fix) {
                 stat.fixableErrorCount++;
             }
@@ -158,6 +162,7 @@ function calculateStatsPerFile(messages) {
         return stat;
     }, {
         errorCount: 0,
+        fatalErrorCount: 0,
         warningCount: 0,
         fixableErrorCount: 0,
         fixableWarningCount: 0
@@ -173,12 +178,14 @@ function calculateStatsPerFile(messages) {
 function calculateStatsPerRun(results) {
     return results.reduce((stat, result) => {
         stat.errorCount += result.errorCount;
+        stat.fatalErrorCount += result.fatalErrorCount;
         stat.warningCount += result.warningCount;
         stat.fixableErrorCount += result.fixableErrorCount;
         stat.fixableWarningCount += result.fixableWarningCount;
         return stat;
     }, {
         errorCount: 0,
+        fatalErrorCount: 0,
         warningCount: 0,
         fixableErrorCount: 0,
         fixableWarningCount: 0

lib/cli.js

@@ -124,18 +124,20 @@ function translateOptions({
 /**
  * Count error messages.
  * @param {LintResult[]} results The lint results.
- * @returns {{errorCount:number;warningCount:number}} The number of error messages.
+ * @returns {{errorCount:number;fatalErrorCount:number;warningCount:number}} The number of error messages.
  */
 function countErrors(results) {
     let errorCount = 0;
+    let fatalErrorCount = 0;
     let warningCount = 0;
 
     for (const result of results) {
         errorCount += result.errorCount;
+        fatalErrorCount += result.fatalErrorCount;
         warningCount += result.warningCount;
     }
 
-    return { errorCount, warningCount };
+    return { errorCount, fatalErrorCount, warningCount };
 }
 
 /**
@@ -305,16 +307,25 @@ const cli = {
         }
 
         if (await printResults(engine, results, options.format, options.outputFile)) {
-            const { errorCount, warningCount } = countErrors(results);
+            const { errorCount, fatalErrorCount, warningCount } = countErrors(results);
             const tooManyWarnings =
                 options.maxWarnings >= 0 && warningCount > options.maxWarnings;
+            const tooManyFatalErrors =
+                options.maxFatalErrors >= 0 && fatalErrorCount > options.maxFatalErrors;
 
             if (!errorCount && tooManyWarnings) {
                 log.error(
                     "ESLint found too many warnings (maximum: %s).",
                     options.maxWarnings
                 );
             }
+            if (tooManyFatalErrors) {
+                log.error(
+                    "ESLint found too many fatal parsing errors (maximum: %s).",
+                    options.maxFatalErrors
+                );
+                return 3;
+            }
 
             return (errorCount || tooManyWarnings) ? 1 : 0;
         }

lib/eslint/eslint.js

@@ -73,6 +73,7 @@ const { version } = require("../../package.json");
  * @property {string} filePath The path to the file that was linted.
  * @property {LintMessage[]} messages All of the messages for the result.
  * @property {number} errorCount Number of errors for the result.
+ * @property {number} fatalErrorCount Number of fatal errors for the result.
  * @property {number} warningCount Number of warnings for the result.
  * @property {number} fixableErrorCount Number of fixable errors for the result.
  * @property {number} fixableWarningCount Number of fixable warnings for the result.

lib/options.js

@@ -157,6 +157,12 @@ module.exports = optionator({
             default: "-1",
             description: "Number of warnings to trigger nonzero exit code"
         },
+        {
+            option: "max-fatal-errors",
+            type: "Int",
+            default: "-1",
+            description: "Number of fatal errors to trigger exit code 3"
+        },
         {
             heading: "Output"
         },

tests/fixtures/max-fatal-errors/.eslintrc

@@ -0,0 +1,3 @@
+{
+    "root": true
+}

tests/fixtures/max-fatal-errors/reserved-keyword.js

@@ -0,0 +1 @@
+import Foo from 'Foo';

tests/lib/cli.js

@@ -785,6 +785,38 @@ describe("cli", () => {
         });
     });
 
+    describe("when given the max-fatal-errors flag", () => {
+        it("should exit with exit code 1 if fatal error count under threshold", async () => {
+            const filePath = getFixturePath("max-fatal-errors");
+            const exitCode = await cli.execute(`--no-ignore --max-fatal-errors 10 ${filePath}`);
+
+            assert.strictEqual(exitCode, 1);
+        });
+
+        it("should exit with exit code 3 if fatal errors count exceeds threshold", async () => {
+            const filePath = getFixturePath("max-fatal-errors");
+            const exitCode = await cli.execute(`--no-ignore --max-fatal-errors 0 ${filePath}`);
+
+            assert.strictEqual(exitCode, 3);
+            assert.ok(log.error.calledOnce);
+            assert.include(log.error.getCall(0).args[0], "ESLint found too many fatal parsing errors");
+        });
+
+        it("should exit with exit code 1 if fatal error count equals threshold", async () => {
+            const filePath = getFixturePath("max-fatal-errors");
+            const exitCode = await cli.execute(`--no-ignore --max-warnings 1 ${filePath}`);
+
+            assert.strictEqual(exitCode, 1);
+        });
+
+        it("should exit with exit code 1 if flag is not specified and there are fatal errors", async () => {
+            const filePath = getFixturePath("max-fatal-errors");
+            const exitCode = await cli.execute(filePath);
+
+            assert.strictEqual(exitCode, 1);
+        });
+    });
+
     describe("when passed --no-inline-config", () => {
         let localCLI;
 

tests/lib/options.js

@@ -311,6 +311,26 @@ describe("options", () => {
         });
     });
 
+    describe("--max-fatal-errors", () => {
+        it("should return correct value for .maxFatalErrors when passed", () => {
+            const currentOptions = options.parse("--max-fatal-errors 10");
+
+            assert.strictEqual(currentOptions.maxFatalErrors, 10);
+        });
+
+        it("should return -1 for .maxFatalErrors when not passed", () => {
+            const currentOptions = options.parse("");
+
+            assert.strictEqual(currentOptions.maxFatalErrors, -1);
+        });
+
+        it("should throw an error when supplied with a non-integer", () => {
+            assert.throws(() => {
+                options.parse("--max-fatal-errors 10.2");
+            }, /Invalid value for option 'max-fatal-errors' - expected type Int/u);
+        });
+    });
+
     describe("--init", () => {
         it("should return true for --init when passed", () => {
             const currentOptions = options.parse("--init");