diff --git a/HISTORY.md b/HISTORY.md index a414578..c03a288 100755 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,9 @@ -### v0.2.0-1 +### v0.2.0-2 + * Added -d (--daemon) switch + * Added helper methods for setting/adding request headers + * Nested requests have cookies/referer automatically set + +### v0.2.0-1 * Added new DOM element getters - innerHTML, rawtext and striptags * Added the ability to specify a custom $ context - $(select, [context]) * Added odd() and even() traversal methods @@ -9,7 +14,7 @@ * Speed improvements * Added Makefile (test / test-cov) -### v0.1.1-17 +### v0.1.1-17 * Fixed incorrect handling of large streams * Better support for request timeouts * Bug fixes diff --git a/README.md b/README.md index 5645ccd..627bf1a 100755 --- a/README.md +++ b/README.md @@ -55,10 +55,7 @@ Check [@nodeio](http://twitter.com/nodeio) or [http://node.io/](http://node.io/) ## Roadmap - Fix up the [http://node.io/](http://node.io/) site -- Nested requests inherit referrer / cookies / user-agent if to the same domain -- `-d,--daemon` node.io switch - Add more DOM [selector](http://api.jquery.com/category/selectors/) / [traversal](http://api.jquery.com/category/traversing/) methods - - ..or attempt a full port of jQuery that's compatible with [htmlparser](https://github.com/tautologistics/node-htmlparser) (I know a port already exists, but it uses the far less forgiving [JSDOM](https://github.com/tmpvar/jsdom)) - Test various proxies and write the proxy documentation - Add distributed processing - Installation without NPM (install.sh) diff --git a/docs/api.md b/docs/api.md index 63419cf..6d02547 100755 --- a/docs/api.md +++ b/docs/api.md @@ -228,17 +228,25 @@ Example **this.getHtml(url, _[headers]_, callback, _[parse]_)** -The same as above, except callback takes `err, $, data, headers` where `$` is the dom selector / traversal object (see DOM selection / traversal below) +The same as above, except callback takes `err, $, data, headers` where `$` is the DOM selector object (see DOM selection / traversal below) -**this.post(url, body, _[headers]_, callback, _[parse]_)** +Example -***this.postHtml(url, body, _[headers]_, callback, _[parse]_)** + this.getHtml('http://www.google.com/', function(err, $, data, headers) { + $('a').each('href', function (href) { + //Print all links on the page + console.log(href); + }); + }); -Makes a POST request. If body is an object, it is encoded using the builtin querystring module. postHtml returns the `$` object. +There are also methods to make post requests. If `body` is an object, it is encoded using the built-in querystring module + + this.post(url, body, [headers], callback, [parse]) + this.postHtml(url, body, [headers], callback, [parse]) -**this.doRequest(method, url, body, _[headers]_, callback, _[parse]_)** +To make a custom request, use the lower level doRequest() method -Makes general a request with the specified options. + this.doRequest(method, url, body, [headers], callback, [parse]) ## Making proxied requests diff --git a/lib/node.io/cli.js b/lib/node.io/cli.js index 8923dee..2d2909e 100755 --- a/lib/node.io/cli.js +++ b/lib/node.io/cli.js @@ -16,19 +16,20 @@ var exit = function (msg, is_error) { }; var usage = '' - + '\x1b[1mUsage\x1b[0m: node.io [options] [JOB] [JOB_ARGS]\n' + + '\x1b[1mUsage\x1b[0m: node.io [OPTIONS] [JOB_ARGS]\n' + '\n' - + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve notfound\n' + + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve\n' + '\n' + '\x1b[1mOptions\x1b[0m:\n' - + ' -i, --input [FILE] Read input from FILE\n' - + ' -o, --output [FILE] Write output to FILE\n' + + ' -i, --input Read input from FILE\n' + + ' -o, --output Write output to FILE\n' + ' -s, --silent Hide console status messages\n' + ' -t, --timeout [TIME] Set a timeout for the operation (in seconds)\n' + ' -f, --fork [NUM] Fork NUM workers. If NUM isn\'t specified, a\n' + ' process is spawned for each CPU core\n' + ' -e, --eval [EXP] Evaluate an expression on each line of input\n' + ' e.g. "input.replace(\'\\t\', \',\')"\n' + + ' -d, --daemon Daemonize the process\n' + ' -b, --benchmark Benchmark the operation\n' + ' -g, --debug Debug the operation\n' + ' -v, --version Display the current version\n' @@ -47,6 +48,7 @@ exports.cli = function (args) { var job_path, job_modified = false, arg, input, output, fork, eval, + daemonize, daemon_arg, job_args = [], options = {}; if (!args.length) { @@ -105,6 +107,13 @@ exports.cli = function (args) { case '--version': exit('v' + require('./').version); break; + case '-d': + case '--daemon': + if (args.length && args[0][0] !== '-') { + daemon_arg = args.shift(); + } + daemonize = true; + break; default: job_path = arg; if (args.length) { @@ -119,7 +128,13 @@ exports.cli = function (args) { var isMaster = !process.env._CHILD_ID_; var start_processor = function (job_path) { - processor.start(job_path, options); + if (daemonize) { + utils.daemonize(daemon_arg, function () { + processor.start(job_path, options); + }); + } else { + processor.start(job_path, options); + } }; if (!job_modified) { diff --git a/lib/node.io/dom.js b/lib/node.io/dom.js index 9400ac3..ecc397a 100755 --- a/lib/node.io/dom.js +++ b/lib/node.io/dom.js @@ -20,8 +20,7 @@ var Job = require('./job').JobProto, Job.prototype.$ = function (selector, context) { var selected = soupselect(context, selector); if (selected.length === 0) { - this.fail_with("No elements matching '" + selector + "'"); - return; + throw new Error("No elements matching '" + selector + "'"); } else if (selected.length === 1) { selected = selected[0]; this.bindToDomElement(selected); diff --git a/lib/node.io/index.js b/lib/node.io/index.js index 1488660..ddcdd86 100755 --- a/lib/node.io/index.js +++ b/lib/node.io/index.js @@ -9,7 +9,7 @@ var processor = require('./processor'), job = require('./job'); exports = module.exports = { - version: '0.2.0-1', + version: '0.2.0-2', Processor: processor.Processor, JobProto: job.JobProto, //A reference to the underlying Job.prototype JobClass: job.JobClass, //A reference to a new prototype identical to Job.prototype (so Job.prototype isn't modified) diff --git a/lib/node.io/io.js b/lib/node.io/io.js index ca93a1d..d1e75b1 100755 --- a/lib/node.io/io.js +++ b/lib/node.io/io.js @@ -21,6 +21,7 @@ var write_request_id = 1, read_request_id = 1, * @api public */ Job.prototype.input = function (start, num, callback) { + this.debug('Reading from STDIN'); var stream = process.openStdin(); this.inputStream(stream); this.input.apply(this, arguments); @@ -33,6 +34,7 @@ Job.prototype.input = function (start, num, callback) { * @api public */ Job.prototype.output = function (data) { + this.debug('Writing to STDOUT'); this.outputStream(process.stdout, 'stdout'); this.output.apply(this, arguments); }; @@ -71,7 +73,8 @@ Job.prototype.outputStream = function (stream, name) { * @param {String} path * @api public */ -Job.prototype.inputFromFile = function (path) { +Job.prototype.inputFromFile = function (path) { + this.debug('Reading from ' + path); var stream = fs.createReadStream(path, {bufferSize: this.options.read_buffer}); this.inputStream(stream); }; @@ -86,6 +89,8 @@ Job.prototype.inputFromFile = function (path) { Job.prototype.inputFromDirectory = function (path) { var self = this, files = fs.readdirSync(path); + this.debug('Reading files in ' + path); + //Trim trailing slash var trim_slash = function (path) { if (path[path.length - 1] === '/') { @@ -278,6 +283,8 @@ Job.prototype.handleSpecialIO = function () { if (typeof this.output === 'string') { var out_path = this.output; + this.debug('Writing to ' + out_path); + //Write output to the file this.output = function (data) { self.write(out_path, data); diff --git a/lib/node.io/job.js b/lib/node.io/job.js index 4042433..b3913bb 100755 --- a/lib/node.io/job.js +++ b/lib/node.io/job.js @@ -5,8 +5,7 @@ */ var validator = require('validator'), - put = require('./utils').put, - put_default = require('./utils').put_default; + utils = require('./utils'); /** * Default job options @@ -27,6 +26,7 @@ var default_options = { newline: '\n', encoding: 'utf8', proxy: false, + useragent: 'node.io', redirects: 3, retry_request: false, args: [] @@ -42,7 +42,7 @@ var Job = exports.JobProto = function (options) { this.reset(); //Set job options - this.options = put_default(options, default_options); + this.options = utils.put_default(options, default_options); //Add data validation methods var val = new validator.Validator(), self = this; @@ -63,12 +63,16 @@ Job.prototype.reset = function () { this.bytes_read = 0; this.bytes_written = 0; this.bytes_received = 0; + + //Store info about the last and next request + this.last = {}; + this.next = {}; } //Each job creates a new class/prototype so that the underlying Job.prototype is untouched exports.__defineGetter__('JobClass', function () { var JobClass = function (options, methods) { - put(JobClass.prototype, methods); + utils.put(JobClass.prototype, methods); Job.apply(this, [options]); @@ -77,7 +81,7 @@ exports.__defineGetter__('JobClass', function () { }; //Extend job methods - put(JobClass.prototype, Job.prototype); + utils.put(JobClass.prototype, Job.prototype); JobClass.prototype.__super__ = Job.prototype; //Compatability with CoffeeScript <= 0.9.4 inheritance @@ -93,11 +97,11 @@ exports.__defineGetter__('JobClass', function () { }; //Extend parent methods - put(Child.prototype, JobPrototype, methods); + utils.put(Child.prototype, JobPrototype, methods); Child.prototype.__super__ = JobPrototype; //Extend parent options - put_default(options, this.options); + utils.put_default(options, this.options); return new Child(options); }; diff --git a/lib/node.io/processor.js b/lib/node.io/processor.js index 39901e4..614fd48 100755 --- a/lib/node.io/processor.js +++ b/lib/node.io/processor.js @@ -229,7 +229,11 @@ Processor.prototype.loadJob = function (job, callback) { if (path.extname(job) !== '.coffee') { //Let node determine the extension and load - callback(null, job, require(job).job); + try { + callback(null, job, require(job).job); + } catch (e) { + callback('Failed to load job "' + job + '"'); + } } else { @@ -258,6 +262,8 @@ Processor.prototype.loadJob = function (job, callback) { } } + } else if (typeof job === 'undefined') { + callback('No job specified! See `node.io --help` for more information.'); } else { callback('Unknown job type: ' + typeof job); } diff --git a/lib/node.io/request.js b/lib/node.io/request.js index 1f2b0f7..9a4069a 100755 --- a/lib/node.io/request.js +++ b/lib/node.io/request.js @@ -7,7 +7,8 @@ var http = require('http'), urlparse = require('url').parse, query = require('querystring'), - Job = require('./job').JobProto; + Job = require('./job').JobProto, + utils = require('./utils'); /** * The default headers to send when using createClient() @@ -143,7 +144,8 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p //Internally keep track of the # of redirects redirects = redirects || 0; if (redirects > this.options.redirects) { - self.fail_with('redirects'); + callback('redirects'); + //self.fail_with('redirects'); return; } @@ -159,11 +161,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p headers = default_headers; } else { //Add default headers - for (var i in default_headers) { - if (typeof headers[i] === 'undefined') { - headers[i] = default_headers[i]; - } - } + utils.put_default(headers, default_headers); } if (!resource.match(/https?:\/\//)) { @@ -175,7 +173,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p if (!port) { port = (url.protocol === 'http:') ? 80 : 443; } - + var host = http.createClient(port, url.hostname), req_url = url.pathname; @@ -183,6 +181,16 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p headers.host = url.hostname; } + //Add headers set before the doRequest call if from the same host (e.g. cookie, user-agent, referer, etc.) + if (typeof this.last.headers === 'object' && this.last.host === url.hostname) { + utils.put(headers, this.last.headers); + this.last = {}; + } + + //Add headers added by setHeader, setCookie, etc. + utils.put(headers, this.next); + this.next = {}; + if (url.search) { req_url += url.search; } @@ -226,33 +234,8 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p //Watch for errors host.addListener('error', function (connectionException) { if (self.is_complete) return; - - var err; - switch (connectionException.errno) { - case 111: - err = 'ECONNREFUSED'; - break; - case 104: - err = 'ECONNRESET'; - break; - case 12: - err = 'ETIMEOUT'; - break; - case 11: - err = 'DNSFAIL'; - break; - case 4: - err = 'ENOTFOUND'; - break; - case 1: - err = 'EPERM'; - break; - default: - err = 'connection'; - break; - } self.debug('Request failed with: ('+connectionException.errno+') ' + connectionException + ' ('+resource+')'); - self.fail_with(err); + callback(connectionException); cleanup(); }); @@ -262,15 +245,15 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p self.timeout = setTimeout(function() { if (self.is_complete) return; self.debug('Request timed out ('+resource+')'); - self.fail_with('timeout'); + //self.fail_with('timeout'); + callback('timeout'); cleanup(); }, this.options.timeout * 1000); } request.on('response', function (response) { if (self.is_complete) { - cleanup(); - return; + return cleanup(); } response.setEncoding('utf8'); @@ -278,25 +261,31 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p var code = response.statusCode || 200; self.debug('\033[7m'+code+'\033[0m '+resource); - + + //Save the response headers for the next request (if to the same host) + var cookies = response.headers['set-cookie']; + self.last = { + host: url.hostname, + headers: { + referer: resource, + cookie: cookies instanceof Array ? cookies.join('; ') : cookies + } + }; + switch (Math.floor(code/100)) { case 4: - self.fail_with(code); + case 5: + callback(code); return; case 3: if (typeof response.headers.location === 'undefined') { - self.fail_with(code); + callback(code); } else { - headers.referer = resource; self.debug(' \033[7m>\033[0m ' + response.headers.location); self.doRequest(method, response.headers.location, body, headers, callback, parse, ++redirects); } return; - - case 5: - self.fail_with(code); - return; } var body = ''; @@ -304,8 +293,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p self.bytes_received += chunk.length; if (self.is_complete) { - cleanup(); - return; + return cleanup(); } body = body + chunk; @@ -313,8 +301,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p response.on('end', function () { if (self.is_complete) { - cleanup(); - return; + return cleanup(); } var parse_callback = function (err, data) { @@ -332,7 +319,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p } } - host.destroy(); + cleanup(); }); }); @@ -343,6 +330,61 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p } }; +/** + * Sets a header on the next request. + * + * @param {Object|String} key + * @param {String} value + * @api public + */ +Job.prototype.setHeader = function (key, value) { + if (typeof key === 'object') { + utils.put(this.next, key); + } else { + this.next[key.toLowerCase()] = value; + } +}; + +/** + * Sets the Cookie for the next request. + * + * @param {String} cookie + * @api public + */ +Job.prototype.setCookie = function (key, value) { + if (value) { + key = encodeURIComponent(key) + '=' + encodeURIComponent(value); + } + this.setHeader('cookie', key); +}; + +/** + * Sets the User-Agent for the next request. + * + * @param {String} agent + * @api public + */ +Job.prototype.setUserAgent = function (agent) { + this.setHeader('user-agent', agent); +}; + +/** + * Adds a cookie to the next request. + * + * @param {String} key + * @param {String} value + * @api public + */ +Job.prototype.addCookie = function (key, value) { + key = encodeURIComponent(key); + value = encodeURIComponent(value); + if (typeof this.next.cookie !== 'undefined' && this.next.cookie.length) { + this.next.cookie += '; ' + key + '=' + value; + } else { + this.next.cookie = key + '=' + value; + } +}; + /** * Returns the total bytes received by any doRequest() calls. * diff --git a/lib/node.io/spawn.js b/lib/node.io/spawn.js index 3c70b53..c255ec7 100755 --- a/lib/node.io/spawn.js +++ b/lib/node.io/spawn.js @@ -21,6 +21,8 @@ Job.prototype.spawn = function (args, stdin, callback) { stdin = undefined; } + this.debug('Spawning "' + args + '"'); + if (!(args instanceof Array)) { args = args.split(' '); } @@ -62,6 +64,7 @@ Job.prototype.spawn = function (args, stdin, callback) { * @api public */ Job.prototype.exec = function (cmd, callback) { + this.debug('Spawning "' + cmd + '"'); var ops = {cwd: process.cwd()}; if (this.options.timeout) { ops.timeout = this.options.timeout * 1000; diff --git a/lib/node.io/utils.js b/lib/node.io/utils.js index 76a0bf1..f76c08c 100755 --- a/lib/node.io/utils.js +++ b/lib/node.io/utils.js @@ -6,6 +6,7 @@ var cwd = process.cwd(), fs = require('fs'), + daemon = require('daemon'), exec = require('child_process').exec; /** @@ -248,4 +249,80 @@ exports.dataToString = function (data, newline, stringify_array) { str = data + newline; } return str; -}; \ No newline at end of file +}; + +/** + * A method for creating and controlling a node.io daemon. + * + * `arg` can be: + * start = daemonizes the process + * stop = stops the daemon if it is running + * restart = alias for stop -> start + * pid = outputs the daemon's PID if it is running + * log = outputs the daemon's log file (stdout & stderr) + * + * @param {String} arg + * @param {Function} callback + * @api public + */ +exports.daemonize = function (arg, callback) { + var lock_file = '/tmp/nodeio.pid', + log_file = '/tmp/nodeio.log'; + + var start = function () { + daemon.run(log_file, lock_file, function (err) { + if (err) return status('Error starting daemon: ' + err, 'error'); + callback(); + }); + }; + + var stop = function () { + try { + fs.readFileSync(lock_file); + } catch (e) { + return status('Daemon is not running', 'error'); + }; + daemon.stop(lock_file, function (err, pid) { + if (err && err.errno === 3) { + return status('Daemon is not running', 'error'); + } else if (err) { + return status('Error stopping daemon: ' + err.errno, 'error'); + } + status('Successfully stopped daemon with pid: ' + pid, 'ok'); + }); + }; + + switch(arg) { + case 'stop': + stop(); + break; + + case 'restart': + daemon.stop(lock_file, function () { + start(); + }); + break; + + case 'log': + try { + console.log(fs.readFileSync(log_file, 'utf8')); + } catch (e) { + return status('No daemon log file', 'error'); + }; + break; + + case 'pid': + try { + var pid = fs.readFileSync(lock_file, 'utf8'); + fs.statSync('/proc/' + pid); + status(pid, 'info'); + } catch (e) { + return status('Daemon is not running', 'error'); + }; + break; + + default: + start(); + break; + } +} \ No newline at end of file diff --git a/package.json b/package.json index b008524..94b57c1 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name" : "node.io", "description" : "A distributed data scraping and processing framework for node.js", - "version" : "0.2.0-1", + "version" : "0.2.0-2", "homepage" : "http://github.com/chriso/node.io", "keywords" : ["data","mapreduce","map","reduce","scraping","html","parsing","parse","scrape","process","processing","data"], "author" : "Chris O'Hara ", @@ -19,7 +19,8 @@ "soupselect": ">= 0.2.0", "validator": ">= 0.1.1", "expresso": ">= 0.7.0", - "coffee-script": ">= 0.9.5" + "coffee-script": ">= 0.9.5", + "daemon": ">= 0.1.0" }, "scripts": { "test": "expresso test" }, "bin": { "node.io": "./bin/node.io" }, diff --git a/test/dom.test.js b/test/dom.test.js index b32f940..6378257 100755 --- a/test/dom.test.js +++ b/test/dom.test.js @@ -129,16 +129,20 @@ module.exports = { assert.equal('function', typeof $('#x').children.first); assert.equal('function', typeof $('#x').children.last); assert.equal('child', $('#x').children.filter('.child').attribs['class']); - - job.fail = function() { - assert.ok(true, 'Selector fail as expected'); - } - + //All of these will fail - assert.isUndefined($('#doesntexist')); - assert.isUndefined($('.doesntexist')); - assert.isUndefined($('p.a.b.c.d')); - assert.isUndefined($('p.a').filter('.xyz')); + assert.throws(function () { + $('#doesntexist'); + }); + assert.throws(function () { + $('.doesntexist'); + }); + assert.throws(function () { + $('p.a.b.c.d'); + }); + assert.throws(function () { + $('p.a').filter('.xyz'); + }); }); }); }, diff --git a/test/processor.test.js b/test/processor.test.js index d56d9a4..7ff99e2 100755 --- a/test/processor.test.js +++ b/test/processor.test.js @@ -241,9 +241,13 @@ module.exports = { var job = createJob({ input: [0,1,2], run: function() { + var self = this; this.parseHtml('

', function(err, $) { + if (err) return self.fail(); assert.equal('a', $('p').attribs['class']); - assert.isUndefined($('#doesntexist')); + assert.throws(function () { + $('#doesntexist') + }); }); }, fail: function() { diff --git a/test/request.test.js b/test/request.test.js index 732f5c6..f7c1154 100755 --- a/test/request.test.js +++ b/test/request.test.js @@ -1,22 +1,19 @@ var nodeio = require('node.io'), - processor = new nodeio.Processor(), http = require('http'), - JobClass = nodeio.JobClass, assert = require('assert'); -var job = new JobClass(); +var port = 24510, timeout = 2000; -var i = 24510; - -//Why do these tests fail on linux?! D: - -job.debug = function () {}; - -//Throw a warning on ECONNREFUSED rather than fail the entire test suite -job.fail = function (input, status) { - if (status === 'ECONNREFUSED') { - console.log('\x1B[33mWARNING\x1B[0m: \x1B[31mECONNREFUSED\x1B[0m (see request.test.js)'); - } +function createJob() { + var JobClass = nodeio.JobClass, job = new JobClass(); + job.debug = function () {}; + //Throw a warning on ECONNREFUSED rather than fail the entire test suite + job.fail = function (input, status) { + if (status === 'ECONNREFUSED') { + console.log('\x1B[33mWARNING\x1B[0m: \x1B[31mECONNREFUSED\x1B[0m (see request.test.js)'); + } + }; + return job; } function close (server) { @@ -29,14 +26,17 @@ module.exports = { 'test GET request': function() { + var job = createJob(); + var server = http.createServer(function (req, res) { - res.writeHead(200,{'Content-Type': 'text/plain'}); + res.writeHead(200, {'Content-Type': 'text/plain'}); res.end('Hello World'); }); - server.listen(++i); - - job.get('http://127.0.0.1:'+i+'/', function(err, data, headers) { + server.listen(++port); + + job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) { + if (err) throw err; assert.equal('text/plain', headers['content-type']); assert.equal('Hello World', data); close(server); @@ -44,11 +44,13 @@ module.exports = { setTimeout(function() { close(server); - }, 1000); + }, timeout); }, 'test GET request with custom headers': function() { + var job = createJob(); + var server = http.createServer(function (req, res) { if (req.headers.foo === 'bar') { res.writeHead(200,{'Content-Type': 'text/plain'}); @@ -58,9 +60,9 @@ module.exports = { } }); - server.listen(++i); + server.listen(++port); - job.get('http://127.0.0.1:'+i+'/', {foo:'bar'}, function(err, data, headers) { + job.get('http://127.0.0.1:'+port+'/', {foo:'bar'}, function(err, data, headers) { assert.equal('text/plain', headers['content-type']); assert.equal('Headers ok', data); close(server); @@ -68,23 +70,25 @@ module.exports = { setTimeout(function() { close(server); - }, 1000); + }, timeout); }, 'test GET request with pre-parse callback': function() { + var job = createJob(); + var server = http.createServer(function (req, res) { res.writeHead(200,{'Content-Type': 'text/plain'}); res.end('><'); }); - server.listen(++i); + server.listen(++port); var parse = function(str) { return str.replace('>','>').replace('<','<'); } - job.get('http://127.0.0.1:'+i+'/', function(err, data, headers) { + job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) { assert.equal('text/plain', headers['content-type']); assert.equal('><', data); close(server); @@ -92,11 +96,13 @@ module.exports = { setTimeout(function() { close(server); - }, 1000); + }, timeout); }, 'test POST request': function() { + var job = createJob(); + var server = http.createServer(function (req, res) { var data = ''; req.setEncoding('utf8'); @@ -111,9 +117,9 @@ module.exports = { }); }); - server.listen(++i); + server.listen(++port); - job.post('http://127.0.0.1:'+i+'/', {foo:'bar'}, function(err, data, headers) { + job.post('http://127.0.0.1:'+port+'/', {foo:'bar'}, function(err, data, headers) { assert.equal('text/plain', headers['content-type']); assert.equal('Post ok', data); close(server); @@ -121,19 +127,21 @@ module.exports = { setTimeout(function() { close(server); - }, 1000); + }, timeout); }, 'test GET request returning the dom': function() { + var job = createJob(); + var server = http.createServer(function (req, res) { res.writeHead(200,{'Content-Type': 'text/plain'}); res.end('

'); }); - server.listen(++i); + server.listen(++port); - job.getHtml('http://127.0.0.1:'+i+'/', function(err, $, data, headers) { + job.getHtml('http://127.0.0.1:'+port+'/', function(err, $, data, headers) { assert.equal('text/plain', headers['content-type']); assert.equal('

', data); assert.equal('a', $('p').attribs['class']); @@ -142,11 +150,13 @@ module.exports = { setTimeout(function() { close(server); - }, 1000); + }, timeout); }, 'test POST request returning the dom': function() { + var job = createJob(); + var server = http.createServer(function (req, res) { var data = ''; req.setEncoding('utf8'); @@ -161,9 +171,9 @@ module.exports = { }); }); - server.listen(++i); + server.listen(++port); - job.postHtml('http://127.0.0.1:'+i+'/', {foo:'bar'}, function(err, $, data, headers) { + job.postHtml('http://127.0.0.1:'+port+'/', {foo:'bar'}, function(err, $, data, headers) { assert.equal('text/plain', headers['content-type']); assert.equal('

', data); assert.equal('a', $('p').attribs['class']); @@ -172,7 +182,97 @@ module.exports = { setTimeout(function() { close(server); - }, 1000); - } + }, timeout); + }, + 'test nested request': function() { + + var p = ++port, job = createJob(); + + var server = http.createServer(function (req, res) { + if (!req.headers.cookie) { + res.writeHead(200, {'Content-Type': 'text/plain', 'Set-Cookie': 'foo=bar'}); + res.end('Ok'); + } else if (req.headers.cookie === 'foo=bar' && req.headers.referer === 'http://127.0.0.1:'+p+'/') { + res.writeHead(200, {'Content-Type': 'text/plain'}); + res.end('Ok2'); + } else { + res.end(); + } + }); + + server.listen(p); + + job.get('http://127.0.0.1:'+p+'/', function(err, data, headers) { + assert.equal('Ok', data); + job.get('http://127.0.0.1:'+p+'/', function(err, data, headers) { + assert.equal('Ok2', data); + close(server); + }); + }); + + setTimeout(function() { + close(server); + }, timeout); + }, + + 'test GET request with custom headers 2': function() { + + var job = createJob(); + + var server = http.createServer(function (req, res) { + if (req.headers.foo === 'bar' && req.headers.cookie === 'coo=kie' && req.headers['user-agent'] === 'Firefox') { + res.writeHead(200, {'Content-Type': 'text/plain'}); + res.end('Headers ok'); + } else { + res.end(); + } + }); + + server.listen(++port); + + job.setHeader('foo', 'bar'); + job.setCookie('coo', 'kie'); + job.setUserAgent('Firefox'); + + job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) { + assert.equal('text/plain', headers['content-type']); + assert.equal('Headers ok', data); + close(server); + }); + + setTimeout(function() { + close(server); + }, timeout); + }, + + 'test GET request with addCookie': function() { + + var job = createJob(); + + var server = http.createServer(function (req, res) { + var cookies = req.headers.cookie.split('; '); + if (cookies[0] === 'coo=kie' && cookies[1] === 'foo=bar') { + res.writeHead(200, {'Content-Type': 'text/plain'}); + res.end('Headers ok'); + } else { + res.end(); + } + }); + + server.listen(++port); + + job.addCookie('coo', 'kie'); + job.addCookie('foo', 'bar'); + + job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) { + assert.equal('text/plain', headers['content-type']); + assert.equal('Headers ok', data); + close(server); + }); + + setTimeout(function() { + close(server); + }, timeout); + }, }