Skip to content

Loading…

Reduction to open file counts #102

Merged
merged 1 commit into from

2 participants

@bryankaplan

Nodemon was sometimes failing to notice when files changed. After some debugging, I discovered that it was unnecessarily trying to open a vast number of files that exceeded the ulimit maximum. Specifically, the following issues have been fixed:

  1. fs.watch was previously watching files matching .nodemonignore regexes. Listing a file in .nodemonignore only prevented nodemon from restarting, but not from opening the file via fs.watch. This created a potentially enormous number of unnecessarily open files from directories like .git and node_modules. This commit fixes that behavior by filtering out the ignored files before handing them to fs.watch.

  2. fs.watch was previously opening each file each time nodemon restarted, without closing the previously open files. That quickly added up to an enormous number of open files. This commit fixes that behavior by keeping an array of watched files, and only requesting fs.watch for files not already in that array.

  3. The above two conditions went unnoticed because nodemon silently did catch EMFILE errors. A new console.error message has been added to identify when too many files are open for watching. After the first two fixes, such situations should be rare, but in larger projects it's easy to trigger by omitting */.git/* and */node_modules/* from .nodemonignore. Prior to the above two fixes, I was seeing hundreds of caught EMFILE errors each time nodemon restarted.

@bryankaplan bryankaplan Reduction to open file counts: No longer fs.watch()ing ignored files,…
… no longer attempting to watch the same files multiple times, and outputing error when ulimit is too low.
469c7d2
@remy remy merged commit d0ba364 into remy:master
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Jun 5, 2012
  1. @bryankaplan

    Reduction to open file counts: No longer fs.watch()ing ignored files,…

    bryankaplan committed
    … no longer attempting to watch the same files multiple times, and outputing error when ulimit is too low.
Showing with 47 additions and 42 deletions.
  1. +47 −42 nodemon.js
View
89 nodemon.js
@@ -36,7 +36,8 @@ var fs = require('fs'),
// Flag to distinguish an app crash from intentional killing (used on Windows only for now)
killedAfterChange = false,
// Make this the last call so it can use the variables defined above (specifically isWindows)
- program = getNodemonArgs();
+ program = getNodemonArgs(),
+ watched = [];
// test to see if the version of find being run supports searching by seconds (-mtime -1s -print)
if (noWatch) {
@@ -239,19 +240,27 @@ function startMonitor() {
fs.readdir(dir, function (err, files) {
if (!err) {
+ files = files.
+ map(function (file ) { return path.join(dir, file); }).
+ filter(ignoredFilter);
files.forEach(function (file) {
- var filename = path.join(dir, file);
- fs.stat(filename, function (err, stat) {
- if (!err && stat) {
- if (stat.isDirectory()) {
- fs.realpath(filename, watch);
+ if (-1 === watched.indexOf(file)) {
+ watched.push(file);
+ fs.stat(file, function (err, stat) {
+ if (!err && stat) {
+ if (stat.isDirectory()) {
+ fs.realpath(file, watch);
+ }
}
- }
- });
+ });
+ }
});
}
});
} catch (e) {
+ if ('EMFILE' === e.code) {
+ console.error('EMFILE: Watching too many files.');
+ }
// ignoring this directory, likely it's "My Music"
// or some such windows fangled stuff
}
@@ -267,45 +276,41 @@ function startMonitor() {
changeFunction = function() { util.error("Nodemon error: changeFunction called when it shouldn't be.") }
}
+ // filter ignored files
+ var ignoredFilter = function (file) {
+ // If we are in a Windows machine
+ if (isWindows) {
+ // Break up the file by slashes
+ var fileParts = file.split(/\\/g);
+
+ // Remove the first piece (C:)
+ fileParts.shift();
+
+ // Join the parts together with Unix slashes
+ file = '/' + fileParts.join('/');
+ }
+ return !reIgnoreFiles.test(file);
+ };
+
var isWindows = process.platform === 'win32';
if ((noWatch || watchWorks) && !program.options.forceLegacyWatch) {
changeFunction(lastStarted, function (files) {
if (files.length) {
- // filter ignored files
- if (ignoreFiles.length) {
- files = files.filter(function(file) {
- // If we are in a Windows machine
- if (isWindows) {
- // Break up the file by slashes
- var fileParts = file.split(/\\/g);
-
- // Remove the first piece (C:)
- fileParts.shift();
-
- // Join the parts together with Unix slashes
- file = '/' + fileParts.join('/');
- }
- return !reIgnoreFiles.test(file);
+ if (restartTimer !== null) clearTimeout(restartTimer);
+ restartTimer = setTimeout(function () {
+ if (program.options.verbose) util.log('[nodemon] restarting due to changes...');
+ files.forEach(function (file) {
+ if (program.options.verbose) util.log('[nodemon] ' + file);
});
- }
-
- if (files.length) {
- if (restartTimer !== null) clearTimeout(restartTimer);
- restartTimer = setTimeout(function () {
- if (program.options.verbose) util.log('[nodemon] restarting due to changes...');
- files.forEach(function (file) {
- if (program.options.verbose) util.log('[nodemon] ' + file);
- });
- if (program.options.verbose) util.print('\n\n');
-
- if (child !== null) {
- child.kill(isWindows ? '' : 'SIGUSR2');
- } else {
- startNode();
- }
- }, restartDelay);
- return;
- }
+ if (program.options.verbose) util.print('\n\n');
+
+ if (child !== null) {
+ child.kill(isWindows ? '' : 'SIGUSR2');
+ } else {
+ startNode();
+ }
+ }, restartDelay);
+ return;
}
if (noWatch) setTimeout(startMonitor, timeout);
Something went wrong with that request. Please try again.