Skip to content
This repository
Browse code

0.2.0-2 release - see HISTORY.md

  • Loading branch information...
commit 8d10368428cd73ca211b2a9f5bd1ce4d7add7db6 1 parent 9834de2
Chris O'Hara authored December 08, 2010
9  HISTORY.md
Source Rendered
... ...
@@ -1,4 +1,9 @@
1  
-###  v0.2.0-1
  1
+### v0.2.0-2
  2
+    * Added -d (--daemon) switch
  3
+    * Added helper methods for setting/adding request headers
  4
+    * Nested requests have cookies/referer automatically set
  5
+
  6
+### v0.2.0-1
2 7
     * Added new DOM element getters - innerHTML, rawtext and striptags
3 8
     * Added the ability to specify a custom $ context - $(select, [context])
4 9
     * Added odd() and even() traversal methods
@@ -9,7 +14,7 @@
9 14
     * Speed improvements
10 15
     * Added Makefile (test / test-cov)
11 16
 
12  
-###  v0.1.1-17
  17
+### v0.1.1-17
13 18
     * Fixed incorrect handling of large streams
14 19
     * Better support for request timeouts
15 20
     * Bug fixes    
3  README.md
Source Rendered
@@ -55,10 +55,7 @@ Check [@nodeio](http://twitter.com/nodeio) or [http://node.io/](http://node.io/)
55 55
 ## Roadmap
56 56
 
57 57
 - Fix up the [http://node.io/](http://node.io/) site
58  
-- Nested requests inherit referrer / cookies / user-agent if to the same domain
59  
-- `-d,--daemon` node.io switch
60 58
 - Add more DOM [selector](http://api.jquery.com/category/selectors/) / [traversal](http://api.jquery.com/category/traversing/) methods
61  
-    - ..or attempt a full port of jQuery that's compatible with [htmlparser](https://github.com/tautologistics/node-htmlparser) (I know a port already exists, but it uses the far less forgiving [JSDOM](https://github.com/tmpvar/jsdom))
62 59
 - Test various proxies and write the proxy documentation
63 60
 - Add distributed processing
64 61
 - Installation without NPM (install.sh)
20  docs/api.md
Source Rendered
@@ -228,17 +228,25 @@ Example
228 228
 
229 229
 **this.getHtml(url, _[headers]_, callback, _[parse]_)**
230 230
 
231  
-The same as above, except callback takes `err, $, data, headers` where `$` is the dom selector / traversal object (see DOM selection / traversal below)
  231
+The same as above, except callback takes `err, $, data, headers` where `$` is the DOM selector object (see DOM selection / traversal below)
232 232
     
233  
-**this.post(url, body, _[headers]_, callback, _[parse]_)**
  233
+Example
234 234
 
235  
-***this.postHtml(url, body, _[headers]_, callback, _[parse]_)**
  235
+    this.getHtml('http://www.google.com/', function(err, $, data, headers) {
  236
+        $('a').each('href', function (href) {
  237
+            //Print all links on the page
  238
+            console.log(href);
  239
+        });
  240
+    });   
236 241
 
237  
-Makes a POST request. If body is an object, it is encoded using the builtin querystring module. postHtml returns the `$` object.
  242
+There are also methods to make post requests. If `body` is an object, it is encoded using the built-in querystring module
  243
+    
  244
+    this.post(url, body, [headers], callback, [parse])
  245
+    this.postHtml(url, body, [headers], callback, [parse])
238 246
 
239  
-**this.doRequest(method, url, body, _[headers]_, callback, _[parse]_)**
  247
+To make a custom request, use the lower level doRequest() method
240 248
 
241  
-Makes general a request with the specified options. 
  249
+    this.doRequest(method, url, body, [headers], callback, [parse])
242 250
 
243 251
 ## Making proxied requests
244 252
 
25  lib/node.io/cli.js
@@ -16,19 +16,20 @@ var exit = function (msg, is_error) {
16 16
 };
17 17
 
18 18
 var usage = ''
19  
-  + '\x1b[1mUsage\x1b[0m: node.io [options] [JOB] [JOB_ARGS]\n'
  19
+  + '\x1b[1mUsage\x1b[0m: node.io [OPTIONS] <JOB_FILE> [JOB_ARGS]\n'
20 20
   + '\n'
21  
-  + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve notfound\n'
  21
+  + '\x1b[1mExample\x1b[0m: node.io -i domains.txt -s resolve\n'
22 22
   + '\n'
23 23
   + '\x1b[1mOptions\x1b[0m:\n'
24  
-  + '  -i, --input [FILE]     Read input from FILE\n'
25  
-  + '  -o, --output [FILE]    Write output to FILE\n'
  24
+  + '  -i, --input <FILE>     Read input from FILE\n'
  25
+  + '  -o, --output <FILE>    Write output to FILE\n'
26 26
   + '  -s, --silent           Hide console status messages\n'
27 27
   + '  -t, --timeout [TIME]   Set a timeout for the operation (in seconds)\n'
28 28
   + '  -f, --fork [NUM]       Fork NUM workers. If NUM isn\'t specified, a\n'
29 29
   + '                         process is spawned for each CPU core\n'
30 30
   + '  -e, --eval [EXP]       Evaluate an expression on each line of input\n'
31 31
   + '                         e.g. "input.replace(\'\\t\', \',\')"\n'
  32
+  + '  -d, --daemon           Daemonize the process\n'
32 33
   + '  -b, --benchmark        Benchmark the operation\n'
33 34
   + '  -g, --debug            Debug the operation\n'
34 35
   + '  -v, --version          Display the current version\n'
@@ -47,6 +48,7 @@ exports.cli = function (args) {
47 48
     
48 49
     var job_path, job_modified = false,
49 50
         arg, input, output, fork, eval,
  51
+        daemonize, daemon_arg,
50 52
         job_args = [], options = {};
51 53
 
52 54
     if (!args.length) {
@@ -105,6 +107,13 @@ exports.cli = function (args) {
105 107
         case '--version':
106 108
             exit('v' + require('./').version);
107 109
             break;
  110
+        case '-d':
  111
+        case '--daemon':
  112
+            if (args.length && args[0][0] !== '-') {
  113
+                daemon_arg = args.shift();
  114
+            }
  115
+            daemonize = true;
  116
+            break;
108 117
         default:
109 118
             job_path = arg;
110 119
             if (args.length) {
@@ -119,7 +128,13 @@ exports.cli = function (args) {
119 128
     var isMaster = !process.env._CHILD_ID_;
120 129
     
121 130
     var start_processor = function (job_path) {
122  
-        processor.start(job_path, options);
  131
+        if (daemonize) {
  132
+            utils.daemonize(daemon_arg, function () {
  133
+                processor.start(job_path, options);
  134
+            });
  135
+        } else {
  136
+            processor.start(job_path, options);
  137
+        }
123 138
     };
124 139
    
125 140
     if (!job_modified) {
3  lib/node.io/dom.js
@@ -20,8 +20,7 @@ var Job = require('./job').JobProto,
20 20
 Job.prototype.$ = function (selector, context) {
21 21
     var selected = soupselect(context, selector);
22 22
     if (selected.length === 0) {
23  
-        this.fail_with("No elements matching '" + selector + "'");
24  
-        return;
  23
+        throw new Error("No elements matching '" + selector + "'");
25 24
     } else if (selected.length === 1) {
26 25
         selected = selected[0];
27 26
         this.bindToDomElement(selected);
2  lib/node.io/index.js
@@ -9,7 +9,7 @@ var processor = require('./processor'),
9 9
     job = require('./job');
10 10
 
11 11
 exports = module.exports = {
12  
-    version: '0.2.0-1',
  12
+    version: '0.2.0-2',
13 13
     Processor: processor.Processor,
14 14
     JobProto: job.JobProto, //A reference to the underlying Job.prototype
15 15
     JobClass: job.JobClass, //A reference to a new prototype identical to Job.prototype (so Job.prototype isn't modified)
9  lib/node.io/io.js
@@ -21,6 +21,7 @@ var write_request_id = 1, read_request_id = 1,
21 21
  * @api public
22 22
  */
23 23
 Job.prototype.input = function (start, num, callback) {
  24
+    this.debug('Reading from STDIN');
24 25
     var stream = process.openStdin();
25 26
     this.inputStream(stream);
26 27
     this.input.apply(this, arguments);
@@ -33,6 +34,7 @@ Job.prototype.input = function (start, num, callback) {
33 34
  * @api public
34 35
  */
35 36
 Job.prototype.output = function (data) {
  37
+    this.debug('Writing to STDOUT');
36 38
     this.outputStream(process.stdout, 'stdout');
37 39
     this.output.apply(this, arguments);
38 40
 };
@@ -71,7 +73,8 @@ Job.prototype.outputStream = function (stream, name) {
71 73
  * @param {String} path
72 74
  * @api public
73 75
  */
74  
-Job.prototype.inputFromFile = function (path) {        
  76
+Job.prototype.inputFromFile = function (path) {   
  77
+    this.debug('Reading from ' + path);     
75 78
     var stream = fs.createReadStream(path, {bufferSize: this.options.read_buffer});
76 79
     this.inputStream(stream);
77 80
 };
@@ -86,6 +89,8 @@ Job.prototype.inputFromFile = function (path) {
86 89
 Job.prototype.inputFromDirectory = function (path) {
87 90
     var self = this, files = fs.readdirSync(path);
88 91
     
  92
+    this.debug('Reading files in ' + path);
  93
+    
89 94
     //Trim trailing slash
90 95
     var trim_slash = function (path) {
91 96
         if (path[path.length - 1] === '/') {
@@ -278,6 +283,8 @@ Job.prototype.handleSpecialIO = function () {
278 283
     if (typeof this.output === 'string') {
279 284
         var out_path = this.output;
280 285
         
  286
+        this.debug('Writing to ' + out_path);
  287
+        
281 288
         //Write output to the file
282 289
         this.output = function (data) {
283 290
             self.write(out_path, data);
18  lib/node.io/job.js
@@ -5,8 +5,7 @@
5 5
  */
6 6
 
7 7
 var validator = require('validator'),
8  
-    put = require('./utils').put,
9  
-    put_default = require('./utils').put_default;
  8
+    utils = require('./utils');
10 9
 
11 10
 /**
12 11
  * Default job options
@@ -27,6 +26,7 @@ var default_options = {
27 26
     newline: '\n',
28 27
     encoding: 'utf8',
29 28
     proxy: false,
  29
+    useragent: 'node.io',
30 30
     redirects: 3,
31 31
     retry_request: false,
32 32
     args: []
@@ -42,7 +42,7 @@ var Job = exports.JobProto = function (options) {
42 42
     this.reset();
43 43
     
44 44
     //Set job options
45  
-    this.options = put_default(options, default_options);
  45
+    this.options = utils.put_default(options, default_options);
46 46
     
47 47
     //Add data validation methods
48 48
     var val = new validator.Validator(), self = this;
@@ -63,12 +63,16 @@ Job.prototype.reset = function () {
63 63
     this.bytes_read = 0;
64 64
     this.bytes_written = 0;
65 65
     this.bytes_received = 0;
  66
+    
  67
+    //Store info about the last and next request
  68
+    this.last = {};
  69
+    this.next = {};
66 70
 }
67 71
 
68 72
 //Each job creates a new class/prototype so that the underlying Job.prototype is untouched
69 73
 exports.__defineGetter__('JobClass', function () {
70 74
     var JobClass = function (options, methods) {
71  
-        put(JobClass.prototype, methods);
  75
+        utils.put(JobClass.prototype, methods);
72 76
         
73 77
         Job.apply(this, [options]);
74 78
         
@@ -77,7 +81,7 @@ exports.__defineGetter__('JobClass', function () {
77 81
     };
78 82
     
79 83
     //Extend job methods
80  
-    put(JobClass.prototype, Job.prototype);
  84
+    utils.put(JobClass.prototype, Job.prototype);
81 85
     JobClass.prototype.__super__ = Job.prototype;
82 86
     
83 87
     //Compatability with CoffeeScript <= 0.9.4 inheritance
@@ -93,11 +97,11 @@ exports.__defineGetter__('JobClass', function () {
93 97
         };
94 98
         
95 99
         //Extend parent methods
96  
-        put(Child.prototype, JobPrototype, methods);
  100
+        utils.put(Child.prototype, JobPrototype, methods);
97 101
         Child.prototype.__super__ = JobPrototype;
98 102
         
99 103
         //Extend parent options
100  
-        put_default(options, this.options);
  104
+        utils.put_default(options, this.options);
101 105
         
102 106
         return new Child(options);
103 107
     };
8  lib/node.io/processor.js
@@ -229,7 +229,11 @@ Processor.prototype.loadJob = function (job, callback) {
229 229
         if (path.extname(job) !== '.coffee') {
230 230
                         
231 231
             //Let node determine the extension and load
232  
-            callback(null, job, require(job).job);
  232
+            try {
  233
+                callback(null, job, require(job).job);
  234
+            } catch (e) {
  235
+                callback('Failed to load job "' + job + '"');
  236
+            }
233 237
             
234 238
         } else {
235 239
             
@@ -258,6 +262,8 @@ Processor.prototype.loadJob = function (job, callback) {
258 262
             }
259 263
         }
260 264
         
  265
+    } else if (typeof job === 'undefined') {
  266
+        callback('No job specified! See `node.io --help` for more information.');
261 267
     } else {
262 268
         callback('Unknown job type: ' + typeof job);
263 269
     }
142  lib/node.io/request.js
@@ -7,7 +7,8 @@
7 7
 var http = require('http'),
8 8
     urlparse = require('url').parse,
9 9
     query = require('querystring'),
10  
-    Job = require('./job').JobProto;
  10
+    Job = require('./job').JobProto,
  11
+    utils = require('./utils');
11 12
 
12 13
 /**
13 14
  * The default headers to send when using createClient()
@@ -143,7 +144,8 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
143 144
     //Internally keep track of the # of redirects
144 145
     redirects = redirects || 0;
145 146
     if (redirects > this.options.redirects) {
146  
-        self.fail_with('redirects');
  147
+        callback('redirects');
  148
+        //self.fail_with('redirects');
147 149
         return;
148 150
     }
149 151
     
@@ -159,11 +161,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
159 161
         headers = default_headers;
160 162
     } else {
161 163
         //Add default headers
162  
-        for (var i in default_headers) {
163  
-            if (typeof headers[i] === 'undefined') {
164  
-                headers[i] = default_headers[i];
165  
-            }
166  
-        }
  164
+        utils.put_default(headers, default_headers);
167 165
     }
168 166
     
169 167
     if (!resource.match(/https?:\/\//)) {
@@ -175,7 +173,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
175 173
     if (!port) {
176 174
         port = (url.protocol === 'http:') ? 80 : 443;
177 175
     }
178  
-    
  176
+        
179 177
     var host = http.createClient(port, url.hostname),
180 178
         req_url = url.pathname;
181 179
     
@@ -183,6 +181,16 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
183 181
         headers.host = url.hostname;
184 182
     }
185 183
     
  184
+    //Add headers set before the doRequest call if from the same host (e.g. cookie, user-agent, referer, etc.)
  185
+    if (typeof this.last.headers === 'object' && this.last.host === url.hostname) {
  186
+        utils.put(headers, this.last.headers);
  187
+        this.last = {};
  188
+    }
  189
+    
  190
+    //Add headers added by setHeader, setCookie, etc.
  191
+    utils.put(headers, this.next);
  192
+    this.next = {};
  193
+    
186 194
     if (url.search) {
187 195
         req_url += url.search;
188 196
     }
@@ -226,33 +234,8 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
226 234
     //Watch for errors
227 235
     host.addListener('error', function (connectionException) {
228 236
         if (self.is_complete) return;
229  
-        
230  
-        var err;
231  
-        switch (connectionException.errno) {
232  
-        case 111:
233  
-            err = 'ECONNREFUSED';
234  
-            break;
235  
-        case 104:
236  
-            err = 'ECONNRESET';
237  
-            break;
238  
-        case 12:
239  
-            err = 'ETIMEOUT';
240  
-            break;
241  
-        case 11:
242  
-            err = 'DNSFAIL';
243  
-            break;
244  
-        case 4:
245  
-            err = 'ENOTFOUND';
246  
-            break;
247  
-        case 1:
248  
-            err = 'EPERM';
249  
-            break;
250  
-        default:
251  
-            err = 'connection';
252  
-            break;
253  
-        }
254 237
         self.debug('Request failed with: ('+connectionException.errno+') ' + connectionException + ' ('+resource+')');
255  
-        self.fail_with(err);
  238
+        callback(connectionException);
256 239
         cleanup();
257 240
     });
258 241
     
@@ -262,15 +245,15 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
262 245
         self.timeout = setTimeout(function() {
263 246
             if (self.is_complete) return;
264 247
             self.debug('Request timed out ('+resource+')');
265  
-            self.fail_with('timeout');
  248
+            //self.fail_with('timeout');
  249
+            callback('timeout');
266 250
             cleanup();
267 251
         }, this.options.timeout * 1000);
268 252
     }
269 253
     
270 254
     request.on('response', function (response) {
271 255
         if (self.is_complete) {
272  
-            cleanup();
273  
-            return;
  256
+            return cleanup();
274 257
         }
275 258
         
276 259
         response.setEncoding('utf8');
@@ -278,25 +261,31 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
278 261
         var code = response.statusCode || 200;
279 262
         
280 263
         self.debug('\033[7m'+code+'\033[0m '+resource);
281  
-    
  264
+        
  265
+        //Save the response headers for the next request (if to the same host)
  266
+        var cookies = response.headers['set-cookie'];
  267
+        self.last = { 
  268
+            host: url.hostname,
  269
+            headers: {
  270
+                referer: resource,
  271
+                cookie: cookies instanceof Array ? cookies.join('; ') : cookies
  272
+            }
  273
+        };
  274
+        
282 275
         switch (Math.floor(code/100)) {
283 276
             case 4:
284  
-                self.fail_with(code);
  277
+            case 5:
  278
+                callback(code);
285 279
                 return;
286 280
             
287 281
             case 3:
288 282
                 if (typeof response.headers.location === 'undefined') {
289  
-                    self.fail_with(code);
  283
+                    callback(code);
290 284
                 } else {
291  
-                    headers.referer = resource;
292 285
                     self.debug('  \033[7m>\033[0m ' + response.headers.location);
293 286
                     self.doRequest(method, response.headers.location, body, headers, callback, parse, ++redirects);
294 287
                 }
295 288
                 return;
296  
-                
297  
-            case 5:
298  
-                self.fail_with(code);
299  
-                return;
300 289
         }
301 290
         
302 291
         var body = '';
@@ -304,8 +293,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
304 293
             self.bytes_received += chunk.length;
305 294
             
306 295
             if (self.is_complete) {
307  
-                cleanup();
308  
-                return;
  296
+                return cleanup();
309 297
             }
310 298
             
311 299
             body = body + chunk; 
@@ -313,8 +301,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
313 301
         
314 302
         response.on('end', function () {
315 303
             if (self.is_complete) {
316  
-                cleanup();
317  
-                return;
  304
+                return cleanup();
318 305
             }
319 306
             
320 307
             var parse_callback = function (err, data) {
@@ -332,7 +319,7 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
332 319
                 }
333 320
             }
334 321
             
335  
-            host.destroy();
  322
+            cleanup();
336 323
         });
337 324
     });
338 325
     
@@ -344,6 +331,61 @@ Job.prototype.doRequest = function (method, resource, body, headers, callback, p
344 331
 };
345 332
 
346 333
 /**
  334
+ * Sets a header on the next request.
  335
+ *
  336
+ * @param {Object|String} key
  337
+ * @param {String} value
  338
+ * @api public
  339
+ */
  340
+Job.prototype.setHeader = function (key, value) {
  341
+    if (typeof key === 'object') {
  342
+        utils.put(this.next, key);
  343
+    } else {
  344
+        this.next[key.toLowerCase()] = value;
  345
+    }
  346
+};
  347
+
  348
+/**
  349
+ * Sets the Cookie for the next request.
  350
+ *
  351
+ * @param {String} cookie
  352
+ * @api public
  353
+ */
  354
+Job.prototype.setCookie = function (key, value) {
  355
+    if (value) {
  356
+        key = encodeURIComponent(key) + '=' + encodeURIComponent(value);
  357
+    }
  358
+    this.setHeader('cookie', key);
  359
+};
  360
+
  361
+/**
  362
+ * Sets the User-Agent for the next request.
  363
+ *
  364
+ * @param {String} agent
  365
+ * @api public
  366
+ */
  367
+Job.prototype.setUserAgent = function (agent) {
  368
+    this.setHeader('user-agent', agent);
  369
+};
  370
+
  371
+/**
  372
+ * Adds a cookie to the next request.
  373
+ *
  374
+ * @param {String} key
  375
+ * @param {String} value
  376
+ * @api public
  377
+ */
  378
+Job.prototype.addCookie = function (key, value) {
  379
+    key = encodeURIComponent(key);
  380
+    value = encodeURIComponent(value);
  381
+    if (typeof this.next.cookie !== 'undefined' && this.next.cookie.length) {
  382
+        this.next.cookie += '; ' + key + '=' + value;
  383
+    } else {
  384
+        this.next.cookie = key + '=' + value;
  385
+    }
  386
+};
  387
+
  388
+/**
347 389
  * Returns the total bytes received by any doRequest() calls.
348 390
  *
349 391
  * @return {Number} bytes_received
3  lib/node.io/spawn.js
@@ -21,6 +21,8 @@ Job.prototype.spawn = function (args, stdin, callback) {
21 21
         stdin = undefined;
22 22
     }
23 23
     
  24
+    this.debug('Spawning "' + args + '"');
  25
+        
24 26
     if (!(args instanceof Array)) {
25 27
         args = args.split(' ');
26 28
     }
@@ -62,6 +64,7 @@ Job.prototype.spawn = function (args, stdin, callback) {
62 64
  * @api public
63 65
  */
64 66
 Job.prototype.exec = function (cmd, callback) {
  67
+    this.debug('Spawning "' + cmd + '"');
65 68
     var ops = {cwd: process.cwd()};
66 69
     if (this.options.timeout) {
67 70
         ops.timeout = this.options.timeout * 1000;
79  lib/node.io/utils.js
@@ -6,6 +6,7 @@
6 6
 
7 7
 var cwd = process.cwd(),
8 8
     fs = require('fs'),
  9
+    daemon = require('daemon'),
9 10
     exec = require('child_process').exec;
10 11
  
11 12
 /**
@@ -248,4 +249,80 @@ exports.dataToString = function (data, newline, stringify_array) {
248 249
         str = data + newline;
249 250
     }
250 251
     return str;
251  
-};
  252
+};
  253
+
  254
+/**
  255
+ * A method for creating and controlling a node.io daemon.
  256
+ *
  257
+ * `arg` can be:
  258
+ *      start = daemonizes the process
  259
+ *      stop  = stops the daemon if it is running
  260
+ *      restart = alias for stop -> start
  261
+ *      pid = outputs the daemon's PID if it is running
  262
+ *      log = outputs the daemon's log file (stdout & stderr)
  263
+ *
  264
+ * @param {String} arg
  265
+ * @param {Function} callback
  266
+ * @api public
  267
+ */
  268
+exports.daemonize = function (arg, callback) {
  269
+    var lock_file = '/tmp/nodeio.pid',
  270
+        log_file = '/tmp/nodeio.log';
  271
+            
  272
+    var start = function () {
  273
+        daemon.run(log_file, lock_file, function (err) {
  274
+            if (err) return status('Error starting daemon: ' + err, 'error');
  275
+            callback();
  276
+        });
  277
+    };
  278
+    
  279
+    var stop = function () {
  280
+        try {
  281
+            fs.readFileSync(lock_file);
  282
+        } catch (e) {
  283
+            return status('Daemon is not running', 'error');
  284
+        };
  285
+        daemon.stop(lock_file, function (err, pid) {
  286
+            if (err && err.errno === 3) {
  287
+                return status('Daemon is not running', 'error');
  288
+            } else if (err) {
  289
+                return status('Error stopping daemon: ' + err.errno, 'error');
  290
+            }
  291
+            status('Successfully stopped daemon with pid: ' + pid, 'ok');
  292
+        });
  293
+    };
  294
+    
  295
+    switch(arg) {
  296
+	case 'stop':
  297
+		stop();
  298
+		break;
  299
+
  300
+	case 'restart':
  301
+        daemon.stop(lock_file, function () {
  302
+            start();
  303
+        });
  304
+		break;
  305
+    
  306
+    case 'log':
  307
+        try {
  308
+            console.log(fs.readFileSync(log_file, 'utf8'));
  309
+        } catch (e) {
  310
+            return status('No daemon log file', 'error');
  311
+        };
  312
+        break;
  313
+    
  314
+    case 'pid':
  315
+        try {
  316
+            var pid = fs.readFileSync(lock_file, 'utf8');
  317
+            fs.statSync('/proc/' + pid);
  318
+            status(pid, 'info');
  319
+        } catch (e) {
  320
+            return status('Daemon is not running', 'error');
  321
+        };
  322
+        break;
  323
+    
  324
+	default:
  325
+        start();
  326
+        break;
  327
+    }
  328
+}
5  package.json
... ...
@@ -1,6 +1,6 @@
1 1
 { "name"          : "node.io",
2 2
   "description"   : "A distributed data scraping and processing framework for node.js",
3  
-  "version"       : "0.2.0-1",
  3
+  "version"       : "0.2.0-2",
4 4
   "homepage"      : "http://github.com/chriso/node.io",
5 5
   "keywords"      : ["data","mapreduce","map","reduce","scraping","html","parsing","parse","scrape","process","processing","data"],
6 6
   "author"        : "Chris O'Hara <cohara87@gmail.com>",
@@ -19,7 +19,8 @@
19 19
     "soupselect": ">= 0.2.0",
20 20
     "validator": ">= 0.1.1",  
21 21
     "expresso": ">= 0.7.0",
22  
-    "coffee-script": ">= 0.9.5"
  22
+    "coffee-script": ">= 0.9.5",
  23
+    "daemon": ">= 0.1.0"
23 24
   },
24 25
   "scripts": { "test": "expresso test" },
25 26
   "bin": { "node.io": "./bin/node.io" },
22  test/dom.test.js
@@ -129,16 +129,20 @@ module.exports = {
129 129
                 assert.equal('function', typeof $('#x').children.first);
130 130
                 assert.equal('function', typeof $('#x').children.last);
131 131
                 assert.equal('child', $('#x').children.filter('.child').attribs['class']);
132  
-                
133  
-                job.fail = function() {
134  
-                    assert.ok(true, 'Selector fail as expected');
135  
-                }
136  
-                
  132
+                                
137 133
                 //All of these will fail
138  
-                assert.isUndefined($('#doesntexist'));
139  
-                assert.isUndefined($('.doesntexist'));
140  
-                assert.isUndefined($('p.a.b.c.d'));
141  
-                assert.isUndefined($('p.a').filter('.xyz'));
  134
+                assert.throws(function () {
  135
+                    $('#doesntexist');
  136
+                });
  137
+                assert.throws(function () {
  138
+                    $('.doesntexist');
  139
+                });
  140
+                assert.throws(function () {
  141
+                    $('p.a.b.c.d');
  142
+                });
  143
+                assert.throws(function () {
  144
+                    $('p.a').filter('.xyz');
  145
+                });
142 146
             });
143 147
         });
144 148
     },
6  test/processor.test.js
@@ -241,9 +241,13 @@ module.exports = {
241 241
         var job = createJob({
242 242
             input: [0,1,2],
243 243
             run: function() {
  244
+                var self = this;
244 245
                 this.parseHtml('<p class="a"></p>', function(err, $) {
  246
+                    if (err) return self.fail();
245 247
                     assert.equal('a', $('p').attribs['class']);
246  
-                    assert.isUndefined($('#doesntexist'));
  248
+                    assert.throws(function () {
  249
+                        $('#doesntexist')
  250
+                    });
247 251
                 });
248 252
             },
249 253
             fail: function() {
170  test/request.test.js
... ...
@@ -1,22 +1,19 @@
1 1
 var nodeio = require('node.io'),
2  
-    processor = new nodeio.Processor(),
3 2
     http = require('http'),
4  
-    JobClass = nodeio.JobClass,
5 3
     assert = require('assert');
6 4
     
7  
-var job = new JobClass();
  5
+var port = 24510, timeout = 2000;
8 6
 
9  
-var i = 24510;
10  
-
11  
-//Why do these tests fail on linux?! D:
12  
-
13  
-job.debug = function () {};
14  
-
15  
-//Throw a warning on ECONNREFUSED rather than fail the entire test suite
16  
-job.fail = function (input, status) {
17  
-    if (status === 'ECONNREFUSED') {
18  
-        console.log('\x1B[33mWARNING\x1B[0m: \x1B[31mECONNREFUSED\x1B[0m (see request.test.js)');
19  
-    }
  7
+function createJob() {
  8
+    var JobClass = nodeio.JobClass, job = new JobClass();
  9
+    job.debug = function () {};
  10
+    //Throw a warning on ECONNREFUSED rather than fail the entire test suite
  11
+    job.fail = function (input, status) {
  12
+        if (status === 'ECONNREFUSED') {
  13
+            console.log('\x1B[33mWARNING\x1B[0m: \x1B[31mECONNREFUSED\x1B[0m (see request.test.js)');
  14
+        }
  15
+    };
  16
+    return job;
20 17
 }
21 18
 
22 19
 function close (server) {
@@ -29,14 +26,17 @@ module.exports = {
29 26
     
30 27
     'test GET request': function() {
31 28
     
  29
+        var job = createJob();
  30
+    
32 31
         var server = http.createServer(function (req, res) {
33  
-            res.writeHead(200,{'Content-Type': 'text/plain'});
  32
+            res.writeHead(200, {'Content-Type': 'text/plain'});
34 33
             res.end('Hello World');
35 34
         });
36 35
         
37  
-        server.listen(++i);
38  
-                
39  
-        job.get('http://127.0.0.1:'+i+'/', function(err, data, headers) {
  36
+        server.listen(++port);
  37
+        
  38
+        job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) {
  39
+            if (err) throw err;
40 40
             assert.equal('text/plain', headers['content-type']);
41 41
             assert.equal('Hello World', data);
42 42
             close(server);
@@ -44,11 +44,13 @@ module.exports = {
44 44
         
45 45
         setTimeout(function() {
46 46
             close(server);
47  
-        }, 1000);
  47
+        }, timeout);
48 48
     },
49 49
     
50 50
     'test GET request with custom headers': function() {
51 51
     
  52
+        var job = createJob();
  53
+        
52 54
         var server = http.createServer(function (req, res) {
53 55
             if (req.headers.foo === 'bar') {
54 56
                 res.writeHead(200,{'Content-Type': 'text/plain'});
@@ -58,9 +60,9 @@ module.exports = {
58 60
             }
59 61
         });
60 62
         
61  
-        server.listen(++i);
  63
+        server.listen(++port);
62 64
                 
63  
-        job.get('http://127.0.0.1:'+i+'/', {foo:'bar'}, function(err, data, headers) {
  65
+        job.get('http://127.0.0.1:'+port+'/', {foo:'bar'}, function(err, data, headers) {
64 66
             assert.equal('text/plain', headers['content-type']);
65 67
             assert.equal('Headers ok', data);
66 68
             close(server);
@@ -68,23 +70,25 @@ module.exports = {
68 70
         
69 71
         setTimeout(function() {
70 72
             close(server);
71  
-        }, 1000);
  73
+        }, timeout);
72 74
     },
73 75
     
74 76
     'test GET request with pre-parse callback': function() {
75 77
     
  78
+        var job = createJob();
  79
+        
76 80
         var server = http.createServer(function (req, res) {
77 81
             res.writeHead(200,{'Content-Type': 'text/plain'});
78 82
             res.end('&gt;&lt;');
79 83
         });
80 84
         
81  
-        server.listen(++i);
  85
+        server.listen(++port);
82 86
         
83 87
         var parse = function(str) {
84 88
             return str.replace('&gt;','>').replace('&lt;','<');
85 89
         }
86 90
         
87  
-        job.get('http://127.0.0.1:'+i+'/', function(err, data, headers) {
  91
+        job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) {
88 92
             assert.equal('text/plain', headers['content-type']);
89 93
             assert.equal('><', data);
90 94
             close(server);
@@ -92,11 +96,13 @@ module.exports = {
92 96
         
93 97
         setTimeout(function() {
94 98
             close(server);
95  
-        }, 1000);
  99
+        }, timeout);
96 100
     },
97 101
     
98 102
     'test POST request': function() {
99 103
     
  104
+        var job = createJob();
  105
+        
100 106
         var server = http.createServer(function (req, res) {
101 107
             var data = '';
102 108
             req.setEncoding('utf8');
@@ -111,9 +117,9 @@ module.exports = {
111 117
             });
112 118
         });
113 119
         
114  
-        server.listen(++i);
  120
+        server.listen(++port);
115 121
                 
116  
-        job.post('http://127.0.0.1:'+i+'/', {foo:'bar'}, function(err, data, headers) {
  122
+        job.post('http://127.0.0.1:'+port+'/', {foo:'bar'}, function(err, data, headers) {
117 123
             assert.equal('text/plain', headers['content-type']);
118 124
             assert.equal('Post ok', data);
119 125
             close(server);
@@ -121,19 +127,21 @@ module.exports = {
121 127
         
122 128
         setTimeout(function() {
123 129
             close(server);
124  
-        }, 1000);
  130
+        }, timeout);
125 131
     },
126 132
     
127 133
     'test GET request returning the dom': function() {
128 134
     
  135
+        var job = createJob();
  136
+        
129 137
         var server = http.createServer(function (req, res) {
130 138
             res.writeHead(200,{'Content-Type': 'text/plain'});
131 139
             res.end('<p class="a"></p>');
132 140
         });
133 141
         
134  
-        server.listen(++i);
  142
+        server.listen(++port);
135 143
                 
136  
-        job.getHtml('http://127.0.0.1:'+i+'/', function(err, $, data, headers) {
  144
+        job.getHtml('http://127.0.0.1:'+port+'/', function(err, $, data, headers) {
137 145
             assert.equal('text/plain', headers['content-type']);
138 146
             assert.equal('<p class="a"></p>', data);
139 147
             assert.equal('a', $('p').attribs['class']);
@@ -142,11 +150,13 @@ module.exports = {
142 150
         
143 151
         setTimeout(function() {
144 152
             close(server);
145  
-        }, 1000);
  153
+        }, timeout);
146 154
     },
147 155
     
148 156
     'test POST request returning the dom': function() {
149 157
     
  158
+        var job = createJob();
  159
+        
150 160
         var server = http.createServer(function (req, res) {
151 161
             var data = '';
152 162
             req.setEncoding('utf8');
@@ -161,9 +171,9 @@ module.exports = {
161 171
             });
162 172
         });
163 173
         
164  
-        server.listen(++i);
  174
+        server.listen(++port);
165 175
                 
166  
-        job.postHtml('http://127.0.0.1:'+i+'/', {foo:'bar'}, function(err, $, data, headers) {
  176
+        job.postHtml('http://127.0.0.1:'+port+'/', {foo:'bar'}, function(err, $, data, headers) {
167 177
             assert.equal('text/plain', headers['content-type']);
168 178
             assert.equal('<p class="a"></p>', data);
169 179
             assert.equal('a', $('p').attribs['class']);
@@ -172,7 +182,97 @@ module.exports = {
172 182
         
173 183
         setTimeout(function() {
174 184
             close(server);
175  
-        }, 1000);
176  
-    }   
  185
+        }, timeout);
  186
+    },
177 187
     
  188
+    'test nested request': function() {
  189
+    
  190
+        var p = ++port, job = createJob();
  191
+        
  192
+        var server = http.createServer(function (req, res) {
  193
+            if (!req.headers.cookie) {
  194
+                res.writeHead(200, {'Content-Type': 'text/plain', 'Set-Cookie': 'foo=bar'});
  195
+                res.end('Ok');
  196
+            } else if (req.headers.cookie === 'foo=bar' && req.headers.referer === 'http://127.0.0.1:'+p+'/') {
  197
+                res.writeHead(200, {'Content-Type': 'text/plain'});
  198
+                res.end('Ok2');
  199
+            } else {
  200
+                res.end();
  201
+            }
  202
+        });
  203
+                
  204
+        server.listen(p);
  205
+        
  206
+        job.get('http://127.0.0.1:'+p+'/', function(err, data, headers) {
  207
+            assert.equal('Ok', data);
  208
+            job.get('http://127.0.0.1:'+p+'/', function(err, data, headers) {
  209
+                assert.equal('Ok2', data);
  210
+                close(server);
  211
+            });
  212
+        });
  213
+        
  214
+        setTimeout(function() {
  215
+            close(server);
  216
+        }, timeout);
  217
+    },
  218
+    
  219
+    'test GET request with custom headers 2': function() {
  220
+    
  221
+        var job = createJob();
  222
+        
  223
+        var server = http.createServer(function (req, res) {
  224
+            if (req.headers.foo === 'bar' && req.headers.cookie === 'coo=kie' && req.headers['user-agent'] === 'Firefox') {
  225
+                res.writeHead(200, {'Content-Type': 'text/plain'});
  226
+                res.end('Headers ok');
  227
+            } else {
  228
+                res.end();
  229
+            }
  230
+        });
  231
+        
  232
+        server.listen(++port);
  233
+        
  234
+        job.setHeader('foo', 'bar');
  235
+        job.setCookie('coo', 'kie');
  236
+        job.setUserAgent('Firefox');
  237
+        
  238
+        job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) {
  239
+            assert.equal('text/plain', headers['content-type']);
  240
+            assert.equal('Headers ok', data);
  241
+            close(server);
  242
+        });
  243
+        
  244
+        setTimeout(function() {
  245
+            close(server);
  246
+        }, timeout);
  247
+    },
  248
+    
  249
+    'test GET request with addCookie': function() {
  250
+    
  251
+        var job = createJob();
  252
+        
  253
+        var server = http.createServer(function (req, res) {
  254
+            var cookies = req.headers.cookie.split('; ');
  255
+            if (cookies[0] === 'coo=kie' && cookies[1] === 'foo=bar') {
  256
+                res.writeHead(200, {'Content-Type': 'text/plain'});
  257
+                res.end('Headers ok');
  258
+            } else {
  259
+                res.end();
  260
+            }
  261
+        });
  262
+        
  263
+        server.listen(++port);
  264
+        
  265
+        job.addCookie('coo', 'kie');
  266
+        job.addCookie('foo', 'bar');
  267
+        
  268
+        job.get('http://127.0.0.1:'+port+'/', function(err, data, headers) {
  269
+            assert.equal('text/plain', headers['content-type']);
  270
+            assert.equal('Headers ok', data);
  271
+            close(server);
  272
+        });
  273
+        
  274
+        setTimeout(function() {
  275
+            close(server);
  276
+        }, timeout);
  277
+    },
178 278
 }

0 notes on commit 8d10368

Please sign in to comment.
Something went wrong with that request. Please try again.