fix: EMFILE errors (#18313)

* fix: EMFILE errors

fixes #18301

* Move catch handler

* Add intentional EMFILE failure

* Use actual limit on Linux systems in test

* Adjust emfile test limit

* Fix linting error

* Fix test for MacOS

* Up MacOS limit in test

* Move tmp file output directory

* Update .gitignore

Co-authored-by: Francesco Trotta <github@fasttime.org>

---------

Co-authored-by: Francesco Trotta <github@fasttime.org>

.github/workflows/ci.yml

@@ -65,6 +65,8 @@ jobs:
         run: node Makefile mocha
       - name: Fuzz Test
         run: node Makefile fuzz
+      - name: Test EMFILE Handling
+        run: npm run test:emfile
 
   test_on_browser:
     name: Browser Test

lib/eslint/eslint.js

@@ -42,6 +42,7 @@ const {
 const { pathToFileURL } = require("url");
 const { FlatConfigArray } = require("../config/flat-config-array");
 const LintResultCache = require("../cli-engine/lint-result-cache");
+const { Retrier } = require("@humanwhocodes/retry");
 
 /*
  * This is necessary to allow overwriting writeFile for testing purposes.
@@ -851,6 +852,8 @@ class ESLint {
             errorOnUnmatchedPattern
         });
         const controller = new AbortController();
+        const retryCodes = new Set(["ENFILE", "EMFILE"]);
+        const retrier = new Retrier(error => retryCodes.has(error.code));
 
         debug(`${filePaths.length} files found in: ${Date.now() - startTime}ms`);
 
@@ -919,7 +922,7 @@ class ESLint {
                     fixer = message => shouldMessageBeFixed(message, config, fixTypesSet) && originalFix(message);
                 }
 
-                return fs.readFile(filePath, { encoding: "utf8", signal: controller.signal })
+                return retrier.retry(() => fs.readFile(filePath, { encoding: "utf8", signal: controller.signal })
                     .then(text => {
 
                         // fail immediately if an error occurred in another file
@@ -949,11 +952,11 @@ class ESLint {
                         }
 
                         return result;
-                    }).catch(error => {
+                    }))
+                    .catch(error => {
                         controller.abort(error);
                         throw error;
                     });
-
             })
         );
 

package.json

@@ -33,7 +33,8 @@
     "test:browser": "node Makefile.js wdio",
     "test:cli": "mocha",
     "test:fuzz": "node Makefile.js fuzz",
-    "test:performance": "node Makefile.js perf"
+    "test:performance": "node Makefile.js perf",
+    "test:emfile": "node tools/check-emfile-handling.js"
   },
   "gitHooks": {
     "pre-commit": "lint-staged"
@@ -71,6 +72,7 @@
     "@eslint/js": "9.0.0",
     "@humanwhocodes/config-array": "^0.12.3",
     "@humanwhocodes/module-importer": "^1.0.1",
+    "@humanwhocodes/retry": "^0.2.3",
     "@nodelib/fs.walk": "^1.2.8",
     "ajv": "^6.12.4",
     "chalk": "^4.0.0",

tests/fixtures/emfile/eslint.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+    rules: {
+        "no-unused-vars": "error"
+    }
+};

tools/check-emfile-handling.js

@@ -0,0 +1,109 @@
+/**
+ * @fileoverview A utility to test that ESLint doesn't crash with EMFILE/ENFILE errors.
+ * @author Nicholas C. Zakas
+ */
+
+"use strict";
+
+//------------------------------------------------------------------------------
+// Requirements
+//------------------------------------------------------------------------------
+
+const fs = require("fs");
+const { readFile } = require("fs/promises");
+const { execSync } = require("child_process");
+const os = require("os");
+
+//------------------------------------------------------------------------------
+// Helpers
+//------------------------------------------------------------------------------
+
+const OUTPUT_DIRECTORY = "tmp/emfile-check";
+const CONFIG_DIRECTORY = "tests/fixtures/emfile";
+
+/*
+ * Every operating system has a different limit for the number of files that can
+ * be opened at once. This number is meant to be larger than the default limit
+ * on most systems.
+ *
+ * Linux systems typically start at a count of 1024 and may be increased to 4096.
+ * MacOS Sonoma v14.4 has a limit of 10496.
+ * Windows has no hard limit but may be limited by available memory.
+ */
+const DEFAULT_FILE_COUNT = 15000;
+let FILE_COUNT = DEFAULT_FILE_COUNT;
+
+// if the platform isn't windows, get the ulimit to see what the actual limit is
+if (os.platform() !== "win32") {
+    try {
+        FILE_COUNT = parseInt(execSync("ulimit -n").toString().trim(), 10) + 1;
+
+        console.log(`Detected Linux file limit of ${FILE_COUNT}.`);
+
+        // if we're on a Mac, make sure the limit isn't high enough to cause a call stack error
+        if (os.platform() === "darwin") {
+            FILE_COUNT = Math.min(FILE_COUNT, 100000);
+        }
+    } catch {
+
+        // ignore error and use default
+    }
+}
+
+/**
+ * Generates files in a directory.
+ * @returns {void}
+ */
+function generateFiles() {
+
+    fs.mkdirSync(OUTPUT_DIRECTORY, { recursive: true });
+
+    for (let i = 0; i < FILE_COUNT; i++) {
+        const fileName = `file_${i}.js`;
+        const fileContent = `// This is file ${i}`;
+
+        fs.writeFileSync(`${OUTPUT_DIRECTORY}/${fileName}`, fileContent);
+    }
+
+}
+
+/**
+ * Generates an EMFILE error by reading all files in the output directory.
+ * @returns {Promise<Buffer[]>} A promise that resolves with the contents of all files.
+ */
+function generateEmFileError() {
+    return Promise.all(
+        Array.from({ length: FILE_COUNT }, (_, i) => {
+            const fileName = `file_${i}.js`;
+
+            return readFile(`${OUTPUT_DIRECTORY}/${fileName}`);
+        })
+    );
+}
+
+//------------------------------------------------------------------------------
+// Main
+//------------------------------------------------------------------------------
+
+console.log(`Generating ${FILE_COUNT} files in ${OUTPUT_DIRECTORY}...`);
+generateFiles();
+
+console.log("Running ESLint...");
+execSync(`node bin/eslint.js ${OUTPUT_DIRECTORY} -c ${CONFIG_DIRECTORY}/eslint.config.js`, { stdio: "inherit" });
+console.log("✅ No errors encountered running ESLint.");
+
+console.log("Checking that this number of files would cause an EMFILE error...");
+generateEmFileError()
+    .then(() => {
+        throw new Error("EMFILE error not encountered.");
+    })
+    .catch(error => {
+        if (error.code === "EMFILE") {
+            console.log("✅ EMFILE error encountered:", error.message);
+        } else if (error.code === "ENFILE") {
+            console.log("✅ ENFILE error encountered:", error.message);
+        } else {
+            console.error("❌ Unexpected error encountered:", error.message);
+            throw error;
+        }
+    });