Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge pull request #1 from wellcaffeinated/master

Fix for "Error: EMFILE, Too many open files 'js/shjs/lang/sh_python.js'"
  • Loading branch information...
commit 95876ec897128fa6e3ceb99e8ceaede3c19ae85f 2 parents 2925b64 + 779bdf6
@pvorb authored
Showing with 51 additions and 29 deletions.
  1. +3 −1 README.mkd
  2. +48 −28 read-files.js
View
4 README.mkd
@@ -1,6 +1,6 @@
**read-files** extends `fs` by the static method `readFiles`.
-## `fs.readFiles(filenames[, encoding], callback)`
+## `fs.readFiles(filenames[, encoding, batchCount], callback)`
Asynchronously reads the entire contents of the files specified in `filenames`.
@@ -8,6 +8,8 @@ Asynchronously reads the entire contents of the files specified in `filenames`.
`encoding` is a string specifying the files' encodings. The default is "utf8".
+`batchCount` is a number specifying the number of files to process at a time before moving to the next batch (default 100).
+
`callback` is a callback function that is passed two arguments `(err, data)`
where `data` is an object of the form:
View
76 read-files.js
@@ -1,33 +1,53 @@
var fs = require("fs");
if (!fs.readFiles)
-fs.readFiles = function(files, encoding, complete) {
-
- var allData = {};
- var todo = 0;
-
+fs.readFiles = function(files, encoding, batchCount, complete){
+ var allData = {}
+ , settings = {}
+ , args = Array.prototype.slice.call(arguments)
+ ;
+
+ //files is first argument
+ files = args.shift();
// complete is the last argument
- complete = arguments[arguments.length - 1];
- // encoding is the second argument
- if (typeof arguments[2] !== "string")
- encoding = "utf8";
- else
- encoding = arguments[2];
-
- // for each file
- files.forEach(function(file) {
- todo++;
-
- // read it's contents
- fs.readFile(file, encoding, function(err, data) {
- if (err) return complete(err, null);
-
- // store data
- allData[file] = data;
-
- // when ready call `complete`
- if (!--todo)
- complete(null, allData);
- });
+ complete = args.pop();
+ //defaults
+ encoding = 'utf8';
+ batchCount = 100;
+ args.forEach(function(arg){
+ switch(typeof arg){
+ case 'string':
+ encoding = arg;
+ break;
+ case 'number':
+ batchCount = arg;
+ break;
+ }
});
-}
+ //recursively handle batches of files at a time
+ function nextChunk(){
+ var next = files.splice(0,batchCount)
+ , len = next.length
+ ;
+ if(!len){
+ complete(null, allData);
+ return;
+ }
+ next.forEach(function(file, idx){
+ // read it's contents
+ fs.readFile(file, encoding, function(err, data) {
+ if (err){
+ complete(err, null);
+ return;
+ }
+
+ // store data
+ allData[file] = data;
+
+ if(idx == (len-1))
+ nextChunk();
+ });
+ });
+ }
+ nextChunk();
+};
Please sign in to comment.
Something went wrong with that request. Please try again.