0.2.0-2 release - see HISTORY.md

node-js-libs · Dec 8, 2010 · 8d10368 · 8d10368
1 parent 9834de2
commit 8d10368
Show file tree

Hide file tree

Showing 16 changed files with 398 additions and 126 deletions.
diff --git a/HISTORY.md b/HISTORY.md
@@ -1,4 +1,9 @@
-###  v0.2.0-1
+### v0.2.0-2
+    * Added -d (--daemon) switch
+    * Added helper methods for setting/adding request headers
+    * Nested requests have cookies/referer automatically set
+
+### v0.2.0-1
     * Added new DOM element getters - innerHTML, rawtext and striptags
     * Added the ability to specify a custom $ context - $(select, [context])
     * Added odd() and even() traversal methods
@@ -9,7 +14,7 @@
     * Speed improvements
     * Added Makefile (test / test-cov)
 
-###  v0.1.1-17
+### v0.1.1-17
     * Fixed incorrect handling of large streams
     * Better support for request timeouts
     * Bug fixes    

diff --git a/README.md b/README.md
@@ -55,10 +55,7 @@ Check [@nodeio](http://twitter.com/nodeio) or [http://node.io/](http://node.io/)
 ## Roadmap
 
 - Fix up the [http://node.io/](http://node.io/) site
-- Nested requests inherit referrer / cookies / user-agent if to the same domain
-- `-d,--daemon` node.io switch
 - Add more DOM [selector](http://api.jquery.com/category/selectors/) / [traversal](http://api.jquery.com/category/traversing/) methods
-    - ..or attempt a full port of jQuery that's compatible with [htmlparser](https://github.com/tautologistics/node-htmlparser) (I know a port already exists, but it uses the far less forgiving [JSDOM](https://github.com/tmpvar/jsdom))
 - Test various proxies and write the proxy documentation
 - Add distributed processing
 - Installation without NPM (install.sh)

diff --git a/docs/api.md b/docs/api.md
@@ -228,17 +228,25 @@ Example
 
 **this.getHtml(url, _[headers]_, callback, _[parse]_)**
 
-The same as above, except callback takes `err, $, data, headers` where `$` is the dom selector / traversal object (see DOM selection / traversal below)
+The same as above, except callback takes `err, $, data, headers` where `$` is the DOM selector object (see DOM selection / traversal below)
 
-**this.post(url, body, _[headers]_, callback, _[parse]_)**
+Example
 
-***this.postHtml(url, body, _[headers]_, callback, _[parse]_)**
+    this.getHtml('http://www.google.com/', function(err, $, data, headers) {
+        $('a').each('href', function (href) {
+            //Print all links on the page
+            console.log(href);
+        });
+    });   
 
-Makes a POST request. If body is an object, it is encoded using the builtin querystring module. postHtml returns the `$` object.
+There are also methods to make post requests. If `body` is an object, it is encoded using the built-in querystring module
+
+    this.post(url, body, [headers], callback, [parse])
+    this.postHtml(url, body, [headers], callback, [parse])
 
-**this.doRequest(method, url, body, _[headers]_, callback, _[parse]_)**
+To make a custom request, use the lower level doRequest() method
 
-Makes general a request with the specified options. 
+    this.doRequest(method, url, body, [headers], callback, [parse])
 
 ## Making proxied requests
 

diff --git a/lib/node.io/cli.js b/lib/node.io/cli.js
@@ -16,19 +16,20 @@ var exit = function (msg, is_error) {
 };
 
 var usage = ''
-  + '\x1b[1mUsage\x1b[0m: node.io [options] [JOB] [JOB_ARGS]\n'
+  + '\x1b[1mUsage\x1b[0m: node.io [OPTIONS] <JOB_FILE> [JOB_ARGS]\n'
   + '\n'
-  + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve notfound\n'
+  + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve\n'
   + '\n'
   + '\x1b[1mOptions\x1b[0m:\n'
-  + '  -i, --input [FILE]     Read input from FILE\n'
-  + '  -o, --output [FILE]    Write output to FILE\n'
+  + '  -i, --input <FILE>     Read input from FILE\n'
+  + '  -o, --output <FILE>    Write output to FILE\n'
   + '  -s, --silent           Hide console status messages\n'
   + '  -t, --timeout [TIME]   Set a timeout for the operation (in seconds)\n'
   + '  -f, --fork [NUM]       Fork NUM workers. If NUM isn\'t specified, a\n'
   + '                         process is spawned for each CPU core\n'
   + '  -e, --eval [EXP]       Evaluate an expression on each line of input\n'
   + '                         e.g. "input.replace(\'\\t\', \',\')"\n'
+  + '  -d, --daemon           Daemonize the process\n'
   + '  -b, --benchmark        Benchmark the operation\n'
   + '  -g, --debug            Debug the operation\n'
   + '  -v, --version          Display the current version\n'
@@ -47,6 +48,7 @@ exports.cli = function (args) {
 
     var job_path, job_modified = false,
         arg, input, output, fork, eval,
+        daemonize, daemon_arg,
         job_args = [], options = {};
 
     if (!args.length) {
@@ -105,6 +107,13 @@ exports.cli = function (args) {
         case '--version':
             exit('v' + require('./').version);
             break;
+        case '-d':
+        case '--daemon':
+            if (args.length && args[0][0] !== '-') {
+                daemon_arg = args.shift();
+            }
+            daemonize = true;
+            break;
         default:
             job_path = arg;
             if (args.length) {
@@ -119,7 +128,13 @@ exports.cli = function (args) {
     var isMaster = !process.env._CHILD_ID_;
 
     var start_processor = function (job_path) {
-        processor.start(job_path, options);
+        if (daemonize) {
+            utils.daemonize(daemon_arg, function () {
+                processor.start(job_path, options);
+            });
+        } else {
+            processor.start(job_path, options);
+        }
     };
 
     if (!job_modified) {

diff --git a/lib/node.io/dom.js b/lib/node.io/dom.js
@@ -20,8 +20,7 @@ var Job = require('./job').JobProto,
 Job.prototype.$ = function (selector, context) {
     var selected = soupselect(context, selector);
     if (selected.length === 0) {
-        this.fail_with("No elements matching '" + selector + "'");
-        return;
+        throw new Error("No elements matching '" + selector + "'");
     } else if (selected.length === 1) {
         selected = selected[0];
         this.bindToDomElement(selected);

diff --git a/lib/node.io/index.js b/lib/node.io/index.js
@@ -9,7 +9,7 @@ var processor = require('./processor'),
     job = require('./job');
 
 exports = module.exports = {
-    version: '0.2.0-1',
+    version: '0.2.0-2',
     Processor: processor.Processor,
     JobProto: job.JobProto, //A reference to the underlying Job.prototype
     JobClass: job.JobClass, //A reference to a new prototype identical to Job.prototype (so Job.prototype isn't modified)

diff --git a/lib/node.io/io.js b/lib/node.io/io.js
@@ -21,6 +21,7 @@ var write_request_id = 1, read_request_id = 1,
  * @api public
  */
 Job.prototype.input = function (start, num, callback) {
+    this.debug('Reading from STDIN');
     var stream = process.openStdin();
     this.inputStream(stream);
     this.input.apply(this, arguments);
@@ -33,6 +34,7 @@ Job.prototype.input = function (start, num, callback) {
  * @api public
  */
 Job.prototype.output = function (data) {
+    this.debug('Writing to STDOUT');
     this.outputStream(process.stdout, 'stdout');
     this.output.apply(this, arguments);
 };
@@ -71,7 +73,8 @@ Job.prototype.outputStream = function (stream, name) {
  * @param {String} path
  * @api public
  */
-Job.prototype.inputFromFile = function (path) {        
+Job.prototype.inputFromFile = function (path) {   
+    this.debug('Reading from ' + path);     
     var stream = fs.createReadStream(path, {bufferSize: this.options.read_buffer});
     this.inputStream(stream);
 };
@@ -86,6 +89,8 @@ Job.prototype.inputFromFile = function (path) {
 Job.prototype.inputFromDirectory = function (path) {
     var self = this, files = fs.readdirSync(path);
 
+    this.debug('Reading files in ' + path);
+
     //Trim trailing slash
     var trim_slash = function (path) {
         if (path[path.length - 1] === '/') {
@@ -278,6 +283,8 @@ Job.prototype.handleSpecialIO = function () {
     if (typeof this.output === 'string') {
         var out_path = this.output;
 
+        this.debug('Writing to ' + out_path);
+
         //Write output to the file
         this.output = function (data) {
             self.write(out_path, data);

diff --git a/lib/node.io/job.js b/lib/node.io/job.js
@@ -5,8 +5,7 @@
  */
 
 var validator = require('validator'),
-    put = require('./utils').put,
-    put_default = require('./utils').put_default;
+    utils = require('./utils');
 
 /**
  * Default job options
@@ -27,6 +26,7 @@ var default_options = {
     newline: '\n',
     encoding: 'utf8',
     proxy: false,
+    useragent: 'node.io',
     redirects: 3,
     retry_request: false,
     args: []
@@ -42,7 +42,7 @@ var Job = exports.JobProto = function (options) {
     this.reset();
 
     //Set job options
-    this.options = put_default(options, default_options);
+    this.options = utils.put_default(options, default_options);
 
     //Add data validation methods
     var val = new validator.Validator(), self = this;
@@ -63,12 +63,16 @@ Job.prototype.reset = function () {
     this.bytes_read = 0;
     this.bytes_written = 0;
     this.bytes_received = 0;
+
+    //Store info about the last and next request
+    this.last = {};
+    this.next = {};
 }
 
 //Each job creates a new class/prototype so that the underlying Job.prototype is untouched
 exports.__defineGetter__('JobClass', function () {
     var JobClass = function (options, methods) {
-        put(JobClass.prototype, methods);
+        utils.put(JobClass.prototype, methods);
 
         Job.apply(this, [options]);
 
@@ -77,7 +81,7 @@ exports.__defineGetter__('JobClass', function () {
     };
 
     //Extend job methods
-    put(JobClass.prototype, Job.prototype);
+    utils.put(JobClass.prototype, Job.prototype);
     JobClass.prototype.__super__ = Job.prototype;
 
     //Compatability with CoffeeScript <= 0.9.4 inheritance
@@ -93,11 +97,11 @@ exports.__defineGetter__('JobClass', function () {
         };
 
         //Extend parent methods
-        put(Child.prototype, JobPrototype, methods);
+        utils.put(Child.prototype, JobPrototype, methods);
         Child.prototype.__super__ = JobPrototype;
 
         //Extend parent options
-        put_default(options, this.options);
+        utils.put_default(options, this.options);
 
         return new Child(options);
     };

diff --git a/lib/node.io/processor.js b/lib/node.io/processor.js
@@ -229,7 +229,11 @@ Processor.prototype.loadJob = function (job, callback) {
         if (path.extname(job) !== '.coffee') {
 
             //Let node determine the extension and load
-            callback(null, job, require(job).job);
+            try {
+                callback(null, job, require(job).job);
+            } catch (e) {
+                callback('Failed to load job "' + job + '"');
+            }
 
         } else {
 
@@ -258,6 +262,8 @@ Processor.prototype.loadJob = function (job, callback) {
             }
         }
 
+    } else if (typeof job === 'undefined') {
+        callback('No job specified! See `node.io --help` for more information.');
     } else {
         callback('Unknown job type: ' + typeof job);
     }