Permalink
Browse files

0.2.0-2 release - see HISTORY.md

  • Loading branch information...
chriso committed Dec 8, 2010
1 parent 9834de2 commit 8d10368428cd73ca211b2a9f5bd1ce4d7add7db6
View
@@ -1,4 +1,9 @@
-### v0.2.0-1
+### v0.2.0-2
+ * Added -d (--daemon) switch
+ * Added helper methods for setting/adding request headers
+ * Nested requests have cookies/referer automatically set
+
+### v0.2.0-1
* Added new DOM element getters - innerHTML, rawtext and striptags
* Added the ability to specify a custom $ context - $(select, [context])
* Added odd() and even() traversal methods
@@ -9,7 +14,7 @@
* Speed improvements
* Added Makefile (test / test-cov)
-### v0.1.1-17
+### v0.1.1-17
* Fixed incorrect handling of large streams
* Better support for request timeouts
* Bug fixes
View
@@ -55,10 +55,7 @@ Check [@nodeio](http://twitter.com/nodeio) or [http://node.io/](http://node.io/)
## Roadmap
- Fix up the [http://node.io/](http://node.io/) site
-- Nested requests inherit referrer / cookies / user-agent if to the same domain
-- `-d,--daemon` node.io switch
- Add more DOM [selector](http://api.jquery.com/category/selectors/) / [traversal](http://api.jquery.com/category/traversing/) methods
- - ..or attempt a full port of jQuery that's compatible with [htmlparser](https://github.com/tautologistics/node-htmlparser) (I know a port already exists, but it uses the far less forgiving [JSDOM](https://github.com/tmpvar/jsdom))
- Test various proxies and write the proxy documentation
- Add distributed processing
- Installation without NPM (install.sh)
View
@@ -228,17 +228,25 @@ Example
**this.getHtml(url, _[headers]_, callback, _[parse]_)**
-The same as above, except callback takes `err, $, data, headers` where `$` is the dom selector / traversal object (see DOM selection / traversal below)
+The same as above, except callback takes `err, $, data, headers` where `$` is the DOM selector object (see DOM selection / traversal below)
-**this.post(url, body, _[headers]_, callback, _[parse]_)**
+Example
-***this.postHtml(url, body, _[headers]_, callback, _[parse]_)**
+ this.getHtml('http://www.google.com/', function(err, $, data, headers) {
+ $('a').each('href', function (href) {
+ //Print all links on the page
+ console.log(href);
+ });
+ });
-Makes a POST request. If body is an object, it is encoded using the builtin querystring module. postHtml returns the `$` object.
+There are also methods to make post requests. If `body` is an object, it is encoded using the built-in querystring module
+
+ this.post(url, body, [headers], callback, [parse])
+ this.postHtml(url, body, [headers], callback, [parse])
-**this.doRequest(method, url, body, _[headers]_, callback, _[parse]_)**
+To make a custom request, use the lower level doRequest() method
-Makes general a request with the specified options.
+ this.doRequest(method, url, body, [headers], callback, [parse])
## Making proxied requests
View
@@ -16,19 +16,20 @@ var exit = function (msg, is_error) {
};
var usage = ''
- + '\x1b[1mUsage\x1b[0m: node.io [options] [JOB] [JOB_ARGS]\n'
+ + '\x1b[1mUsage\x1b[0m: node.io [OPTIONS] <JOB_FILE> [JOB_ARGS]\n'
+ '\n'
- + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve notfound\n'
+ + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve\n'
+ '\n'
+ '\x1b[1mOptions\x1b[0m:\n'
- + ' -i, --input [FILE] Read input from FILE\n'
- + ' -o, --output [FILE] Write output to FILE\n'
+ + ' -i, --input <FILE> Read input from FILE\n'
+ + ' -o, --output <FILE> Write output to FILE\n'
+ ' -s, --silent Hide console status messages\n'
+ ' -t, --timeout [TIME] Set a timeout for the operation (in seconds)\n'
+ ' -f, --fork [NUM] Fork NUM workers. If NUM isn\'t specified, a\n'
+ ' process is spawned for each CPU core\n'
+ ' -e, --eval [EXP] Evaluate an expression on each line of input\n'
+ ' e.g. "input.replace(\'\\t\', \',\')"\n'
+ + ' -d, --daemon Daemonize the process\n'
+ ' -b, --benchmark Benchmark the operation\n'
+ ' -g, --debug Debug the operation\n'
+ ' -v, --version Display the current version\n'
@@ -47,6 +48,7 @@ exports.cli = function (args) {
var job_path, job_modified = false,
arg, input, output, fork, eval,
+ daemonize, daemon_arg,
job_args = [], options = {};
if (!args.length) {
@@ -105,6 +107,13 @@ exports.cli = function (args) {
case '--version':
exit('v' + require('./').version);
break;
+ case '-d':
+ case '--daemon':
+ if (args.length && args[0][0] !== '-') {
+ daemon_arg = args.shift();
+ }
+ daemonize = true;
+ break;
default:
job_path = arg;
if (args.length) {
@@ -119,7 +128,13 @@ exports.cli = function (args) {
var isMaster = !process.env._CHILD_ID_;
var start_processor = function (job_path) {
- processor.start(job_path, options);
+ if (daemonize) {
+ utils.daemonize(daemon_arg, function () {
+ processor.start(job_path, options);
+ });
+ } else {
+ processor.start(job_path, options);
+ }
};
if (!job_modified) {
View
@@ -20,8 +20,7 @@ var Job = require('./job').JobProto,
Job.prototype.$ = function (selector, context) {
var selected = soupselect(context, selector);
if (selected.length === 0) {
- this.fail_with("No elements matching '" + selector + "'");
- return;
+ throw new Error("No elements matching '" + selector + "'");
} else if (selected.length === 1) {
selected = selected[0];
this.bindToDomElement(selected);
View
@@ -9,7 +9,7 @@ var processor = require('./processor'),
job = require('./job');
exports = module.exports = {
- version: '0.2.0-1',
+ version: '0.2.0-2',
Processor: processor.Processor,
JobProto: job.JobProto, //A reference to the underlying Job.prototype
JobClass: job.JobClass, //A reference to a new prototype identical to Job.prototype (so Job.prototype isn't modified)
View
@@ -21,6 +21,7 @@ var write_request_id = 1, read_request_id = 1,
* @api public
*/
Job.prototype.input = function (start, num, callback) {
+ this.debug('Reading from STDIN');
var stream = process.openStdin();
this.inputStream(stream);
this.input.apply(this, arguments);
@@ -33,6 +34,7 @@ Job.prototype.input = function (start, num, callback) {
* @api public
*/
Job.prototype.output = function (data) {
+ this.debug('Writing to STDOUT');
this.outputStream(process.stdout, 'stdout');
this.output.apply(this, arguments);
};
@@ -71,7 +73,8 @@ Job.prototype.outputStream = function (stream, name) {
* @param {String} path
* @api public
*/
-Job.prototype.inputFromFile = function (path) {
+Job.prototype.inputFromFile = function (path) {
+ this.debug('Reading from ' + path);
var stream = fs.createReadStream(path, {bufferSize: this.options.read_buffer});
this.inputStream(stream);
};
@@ -86,6 +89,8 @@ Job.prototype.inputFromFile = function (path) {
Job.prototype.inputFromDirectory = function (path) {
var self = this, files = fs.readdirSync(path);
+ this.debug('Reading files in ' + path);
+
//Trim trailing slash
var trim_slash = function (path) {
if (path[path.length - 1] === '/') {
@@ -278,6 +283,8 @@ Job.prototype.handleSpecialIO = function () {
if (typeof this.output === 'string') {
var out_path = this.output;
+ this.debug('Writing to ' + out_path);
+
//Write output to the file
this.output = function (data) {
self.write(out_path, data);
View
@@ -5,8 +5,7 @@
*/
var validator = require('validator'),
- put = require('./utils').put,
- put_default = require('./utils').put_default;
+ utils = require('./utils');
/**
* Default job options
@@ -27,6 +26,7 @@ var default_options = {
newline: '\n',
encoding: 'utf8',
proxy: false,
+ useragent: 'node.io',
redirects: 3,
retry_request: false,
args: []
@@ -42,7 +42,7 @@ var Job = exports.JobProto = function (options) {
this.reset();
//Set job options
- this.options = put_default(options, default_options);
+ this.options = utils.put_default(options, default_options);
//Add data validation methods
var val = new validator.Validator(), self = this;
@@ -63,12 +63,16 @@ Job.prototype.reset = function () {
this.bytes_read = 0;
this.bytes_written = 0;
this.bytes_received = 0;
+
+ //Store info about the last and next request
+ this.last = {};
+ this.next = {};
}
//Each job creates a new class/prototype so that the underlying Job.prototype is untouched
exports.__defineGetter__('JobClass', function () {
var JobClass = function (options, methods) {
- put(JobClass.prototype, methods);
+ utils.put(JobClass.prototype, methods);
Job.apply(this, [options]);
@@ -77,7 +81,7 @@ exports.__defineGetter__('JobClass', function () {
};
//Extend job methods
- put(JobClass.prototype, Job.prototype);
+ utils.put(JobClass.prototype, Job.prototype);
JobClass.prototype.__super__ = Job.prototype;
//Compatability with CoffeeScript <= 0.9.4 inheritance
@@ -93,11 +97,11 @@ exports.__defineGetter__('JobClass', function () {
};
//Extend parent methods
- put(Child.prototype, JobPrototype, methods);
+ utils.put(Child.prototype, JobPrototype, methods);
Child.prototype.__super__ = JobPrototype;
//Extend parent options
- put_default(options, this.options);
+ utils.put_default(options, this.options);
return new Child(options);
};
View
@@ -229,7 +229,11 @@ Processor.prototype.loadJob = function (job, callback) {
if (path.extname(job) !== '.coffee') {
//Let node determine the extension and load
- callback(null, job, require(job).job);
+ try {
+ callback(null, job, require(job).job);
+ } catch (e) {
+ callback('Failed to load job "' + job + '"');
+ }
} else {
@@ -258,6 +262,8 @@ Processor.prototype.loadJob = function (job, callback) {
}
}
+ } else if (typeof job === 'undefined') {
+ callback('No job specified! See `node.io --help` for more information.');
} else {
callback('Unknown job type: ' + typeof job);
}
Oops, something went wrong.

0 comments on commit 8d10368

Please sign in to comment.