Permalink
Browse files

added synchronous version

  • Loading branch information...
1 parent 6b66729 commit 8d4dc9804ab8c9f9e5bfab21de7bdef682a1ae24 @coolaj86 coolaj86 committed May 3, 2011
Showing with 255 additions and 58 deletions.
  1. +10 −2 README.md
  2. +92 −0 lib/walk-async-only.js
  3. +110 −54 lib/walk.js
  4. +40 −0 testSync.js
  5. +3 −2 walk-test.sh
View
@@ -9,6 +9,7 @@ This is somewhat of a port python's `os.walk`, but using Node.JS conventions.
* Asynchronous
* Chronological (optionally)
* Built-in flow-control
+ * includes Synchronous version (same API as Asynchronous)
As few file descriptors are opened at a time as possible.
This is particularly well suited for single hard disks which are not flash or solid state.
@@ -21,6 +22,10 @@ Installation
Usage
====
+Both Asynchronous and Synchronous versions are provided.
+
+The Synchronous version still uses callbacks, so it is safe to use with other Asynchronous functions and will still work as expected.
+
var walk = require('walk'),
fs = require('fs'),
options,
@@ -30,7 +35,10 @@ Usage
followLinks: false,
};
- walker = walk("/tmp", options);
+ walker = walk.walk("/tmp", options);
+
+ // OR
+ // walker = walk.walkSync("/tmp", options);
walker.on("names", function (root, nodeNamesArray) {
nodeNames.sort(function (a, b) {
@@ -148,4 +156,4 @@ Note that `find.js` omits the start directory
user 0m17.661s
sys 0m24.008s
-In conclusion node.js asynchronous walk is much slower than regular "find".
+In conclusion node.js asynchronous walk is much slower than regular "find".
View
@@ -0,0 +1,92 @@
+(function () {
+ "use strict"
+
+ // Array.prototype.forEachAsync(next, item, i, collection)
+ require('futures/forEachAsync');
+
+ function noop() {}
+
+ var fs = require('fs'),
+ EventEmitter = require('events').EventEmitter,
+ TypeEmitter = require('./node-type-emitter');
+
+ // 2010-11-25 jorge@jorgechamorro.com
+ function create(pathname, cb) {
+ var emitter = new EventEmitter(),
+ q = [],
+ queue = [q],
+ curpath;
+
+ function walk() {
+ fs.readdir(curpath, function(err, files) {
+ if (err) {
+ emitter.emit('directoryError', curpath, { error: err }, noop);
+ //emitter.emit('error', curpath, { error: err });
+ }
+ // XXX bug was here. next() was omitted
+ if (!files || 0 == files.length) {
+ return next();
+ }
+
+ var fnodeGroups = TypeEmitter.createNodeGroups();
+
+ // TODO could allow user to selectively stat
+ // and don't stat if there are no stat listeners
+ emitter.emit('names', curpath, files, noop);
+ files.forEachAsync(function (cont, file) {
+ emitter.emit('name', curpath, file, noop);
+ fs.lstat(curpath + '/' + file, function (err, stat) {
+ stat = stat || {};
+ stat.name = file;
+ if (err) {
+ stat.error = err;
+ //emitter.emit('error', curpath, stat);
+ emitter.emit('nodeError', curpath, stat, noop);
+ fnodeGroups.errors.push(stat);
+ cont();
+ } else {
+ TypeEmitter.sortFnodesByType(stat, fnodeGroups);
+ TypeEmitter.emitNodeType(emitter, curpath, stat, cont);
+ }
+ });
+ }).then(function () {
+ if (fnodeGroups.errors.length) {
+ emitter.emit('errors', curpath, fnodeGroups.errors, noop);
+ }
+ TypeEmitter.emitNodeTypeGroups(emitter, curpath, fnodeGroups, function () {
+ var dirs = [];
+ fnodeGroups.directories.forEach(function (stat) {
+ dirs.push(stat.name);
+ });
+ dirs.forEach(fullPath);
+ queue.push(q = dirs);
+ next();
+ });
+ });
+ });
+ }
+
+ function next() {
+ if (q.length) {
+ curpath = q.pop();
+ return walk();
+ }
+ if (queue.length -= 1) {
+ q = queue[queue.length-1];
+ return next();
+ }
+ emitter.emit('end');
+ }
+
+ function fullPath(v,i,o) {
+ o[i]= [curpath, '/', v].join('');
+ }
+
+ curpath = pathname;
+ walk();
+
+ return emitter;
+ }
+
+ module.exports = create;
+}());
View
@@ -1,3 +1,4 @@
+// Adapted from work by jorge@jorgechamorro.com on 2010-11-25
(function () {
"use strict"
@@ -10,60 +11,104 @@
EventEmitter = require('events').EventEmitter,
TypeEmitter = require('./node-type-emitter');
- // 2010-11-25 jorge@jorgechamorro.com
- function create(pathname, cb) {
- var emitter = new EventEmitter(),
- q = [],
- queue = [q],
- curpath;
+ function create(pathname, options, sync) {
+ var emitter = new EventEmitter()
+ , q = []
+ , queue = [q]
+ , curpath;
- function walk() {
- fs.readdir(curpath, function(err, files) {
- if (err) {
- emitter.emit('directoryError', curpath, { error: err }, noop);
- //emitter.emit('error', curpath, { error: err });
- }
- // XXX bug was here. next() was omitted
- if (!files || 0 == files.length) {
- return next();
+ function readdirHandler(err, files) {
+ var fnodeGroups = TypeEmitter.createNodeGroups();
+
+ function filesHandler(cont, file) {
+ var statPath;
+
+ emitter.emit('name', curpath, file, noop);
+
+ function lstatHandler(err, stat) {
+ stat = stat || {};
+ stat.name = file;
+ if (err) {
+ stat.error = err;
+ //emitter.emit('error', curpath, stat);
+ emitter.emit('nodeError', curpath, stat, noop);
+ fnodeGroups.errors.push(stat);
+ cont();
+ } else {
+ TypeEmitter.sortFnodesByType(stat, fnodeGroups);
+ TypeEmitter.emitNodeType(emitter, curpath, stat, cont);
+ }
}
- var fnodeGroups = TypeEmitter.createNodeGroups();
-
- // TODO could allow user to selectively stat
- // and don't stat if there are no stat listeners
- emitter.emit('names', curpath, files, noop);
- files.forEachAsync(function (cont, file) {
- emitter.emit('name', curpath, file, noop);
- fs.lstat(curpath + '/' + file, function (err, stat) {
- stat = stat || {};
- stat.name = file;
- if (err) {
- stat.error = err;
- //emitter.emit('error', curpath, stat);
- emitter.emit('nodeError', curpath, stat, noop);
- fnodeGroups.errors.push(stat);
- cont();
- } else {
- TypeEmitter.sortFnodesByType(stat, fnodeGroups);
- TypeEmitter.emitNodeType(emitter, curpath, stat, cont);
- }
- });
- }).then(function () {
- if (fnodeGroups.errors.length) {
- emitter.emit('errors', curpath, fnodeGroups.errors, noop);
+ statPath = curpath + '/' + file;
+ if (sync) {
+ try {
+ lstatHandler(null, fs.lstatSync(statPath));
+ } catch(e) {
+ lstatHandler(e);
}
- TypeEmitter.emitNodeTypeGroups(emitter, curpath, fnodeGroups, function () {
- var dirs = [];
- fnodeGroups.directories.forEach(function (stat) {
- dirs.push(stat.name);
- });
- dirs.forEach(fullPath);
- queue.push(q = dirs);
- next();
+ } else {
+ fs.lstat(statPath, lstatHandler);
+ }
+ }
+
+ function postFilesHandler() {
+ if (fnodeGroups.errors.length) {
+ emitter.emit('errors', curpath, fnodeGroups.errors, noop);
+ }
+ TypeEmitter.emitNodeTypeGroups(emitter, curpath, fnodeGroups, function () {
+ var dirs = [];
+ fnodeGroups.directories.forEach(function (stat) {
+ dirs.push(stat.name);
});
+ dirs.forEach(fullPath);
+ queue.push(q = dirs);
+ next();
+ });
+ }
+
+ if (err) {
+ emitter.emit('directoryError', curpath, { error: err }, noop);
+ //emitter.emit('error', curpath, { error: err });
+ }
+
+ if (!files || 0 == files.length) {
+ return next();
+ }
+
+ // TODO could allow user to selectively stat
+ // and don't stat if there are no stat listeners
+ emitter.emit('names', curpath, files, noop);
+
+ if (sync) {
+ files.forEach(function (items) {
+ filesHandler(noop, items);
});
- });
+ postFilesHandler();
+ } else {
+ files.forEachAsync(filesHandler).then(postFilesHandler);
+ }
+ }
+
+ function walkSync() {
+ var err, files;
+
+ try {
+ files = fs.readdirSync(curpath);
+ } catch(e) {
+ err = e;
+ }
+
+ readdirHandler(err, files);
+ }
+
+ function walk() {
+ if (sync) {
+ walkSync();
+ return;
+ }
+
+ fs.readdir(curpath, readdirHandler);
}
function next() {
@@ -72,21 +117,32 @@
return walk();
}
if (queue.length -= 1) {
- q = queue[queue.length-1];
+ q = queue[queue.length - 1];
return next();
}
emitter.emit('end');
}
- function fullPath(v,i,o) {
- o[i]= [curpath, '/', v].join('');
+ function fullPath(v, i, o) {
+ o[i] = [curpath, '/', v].join('');
}
curpath = pathname;
- walk();
-
+
+ if (sync) {
+ process.nextTick(walk);
+ } else {
+ walk();
+ }
+
return emitter;
}
- module.exports = create;
+ exports.walk = function (path, opts) {
+ return create(path, opts, false);
+ };
+
+ exports.walkSync = function (path, opts) {
+ return create(path, opts, true);
+ };
}());
View
@@ -0,0 +1,40 @@
+(function () {
+ "use strict";
+
+ var walk = require('./lib/walk')
+ , _ = require('underscore')
+ , fs = require('fs')
+ , sync = false
+ , walker;
+
+ console.log(walk);
+
+ walker = walk.walk(".");
+
+ walker.on("directory", function (root, dirStatsArray, next) {
+ // dirStatsArray is an array of `stat` objects with the additional attributes
+ // * type
+ // * error
+ // * name
+ //console.log(_.pluck(dirStatsArray, 'name'));
+ console.log(root + '/' + dirStatsArray.name);
+
+ next();
+ });
+
+ walker.on("file", function (root, fileStats, next) {
+ console.log(root + '/' + fileStats.name);
+
+ next();
+ });
+
+ walker.on("errors", function (root, nodeStatsArray, next) {
+ //console.log(nodeStatsArray);
+
+ next();
+ });
+
+ walker.on("end", function () {
+ console.log("all done");
+ });
+}());
View
@@ -1,5 +1,6 @@
mkdir -p walk-test/dir1
+touch walk-test/file-0
touch walk-test/file-1
-touch walk-test/file-2
-touch walk-test/dir1/file-1
touch walk-test/dir1/file-2
+touch walk-test/dir1/file-3
+echo "4 files and 2 directories (including '.')"

0 comments on commit 8d4dc98

Please sign in to comment.