From a3e56c175b20fa1bf072b04c471e46496f498f41 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 17 Nov 2010 07:58:40 +1000 Subject: [PATCH] Updated README and made some JSLint fixes --- README.md | 13 ++++++++----- lib/node.io/job.js | 20 +++++++++----------- lib/node.io/processor.js | 4 +--- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6b3d099..54030e6 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,19 @@ For usage details, run ## Why node.io? -- Seamlessly distribute work across child processes and servers (soon) +- Create modular and extensible jobs for scraping and processing data +- Seamlessly distribute work between child processes and servers (soon) - Handles a variety of input / output situations * Reading / writing lines to and from files + * Reading all files in a directory (and recursing if specified) * To / from a database * STDIN / STDOUT * Piping between node.io jobs - * Any combination of the above + * Custom IO / any combination of the above - Includes a robust framework for scraping and selecting HTML data -- Support for proxied requests -- Provides support for retries and timeouts +- Support for a variety of proxies when making requests +- Includes a data validation and sanitization framework +- Provides support for retries, timeouts, dynamically adding input, etc. ## Examples @@ -33,7 +36,7 @@ Coming soon. See [http://node.io/](http://node.io/) for updates ## Credits -node.io uses the following (awesome) libraries: +node.io uses the following awesome libraries: - [tautologistics'](https://github.com/tautologistics) [node-htmlparser](https://github.com/tautologistics/node-htmlparser) - [harryf's](https://github.com/harryf) [soupselect](https://github.com/harryf/node-soupselect) diff --git a/lib/node.io/job.js b/lib/node.io/job.js index 0673480..505a718 100755 --- a/lib/node.io/job.js +++ b/lib/node.io/job.js @@ -5,7 +5,6 @@ var default_options = { max: 1, take: 1, - timeout: 0, retries: 2, timeout: false, flatten: true, @@ -45,8 +44,8 @@ var Job = exports.Job = function(options, methods) { } Job.prototype.extend = function(options, methods, extend_this) { + var i, job; - var job; if (!extend_this) { job = new Job(this.options, this.methods); } else { @@ -56,14 +55,14 @@ Job.prototype.extend = function(options, methods, extend_this) { //Set default options if (typeof job.options === 'undefined') { job.options = {}; - for (var i in default_options) { + for (i in default_options) { job.options[i] = default_options[i]; } } //Extend/set options with the user specified ops if (typeof options === 'object') { - for (var i in options) { + for (i in options) { job.options[i] = options[i]; } } @@ -71,8 +70,9 @@ Job.prototype.extend = function(options, methods, extend_this) { //Extend/set job methods job.methods = job.methods || {}; if (typeof methods === 'object') { + //TODO: Implement a way of calling parent methods, e.g. _super_() - for(var i in methods) { + for(i in methods) { job.methods[i] = methods[i]; job[i] = methods[i]; } @@ -93,16 +93,16 @@ Job.prototype.cancelPreviousTimeouts = function() { Job.prototype.emit = function() { var self = this; - if (this.after_timeout) return; + if (this.after_timeout) { + return; + } var process_order = ['take','run','output_hook']; this.cancelPreviousTimeouts(); //No methods left? we're done - if (this.stage >= process_order.length) { - return; - } + if (this.stage >= process_order.length) return; //Get the next method in the chain var next = process_order[this.stage++]; @@ -129,8 +129,6 @@ Job.prototype.emit = function() { } Job.prototype.reduce_hook = function(data, callback) { - var self = this; - if (typeof this.reduce !== 'function') { if (callback) callback.apply(this, [data]); return; diff --git a/lib/node.io/processor.js b/lib/node.io/processor.js index c61a9fb..02ecb75 100755 --- a/lib/node.io/processor.js +++ b/lib/node.io/processor.js @@ -47,7 +47,7 @@ exports.start = function(job, options, callback) { var job_obj; if (typeof job === 'string') { if (!~job.indexOf('/')) job = process.cwd() + '/' + job; - job_obj = p.loadJob(job) + job_obj = p.loadJob(job); } else { job_obj = job; job = crc32(JSON.stringify(job)); @@ -558,8 +558,6 @@ Processor.prototype.instanceSpawn = function(job) { } } -var x = y = z = 0; - Processor.prototype.instanceStart = function(job, input) { var self = this, j = this.jobs[job];