Permalink
Browse files

Merge pull request #22 from seanowenhayes/master

Check for connection error in main.
  • Loading branch information...
2 parents 64afb99 + 127ceb7 commit 1ccaa370c9b28efd52159d1919f806f2f6856fc5 @mikeal committed Mar 18, 2013
Showing with 46 additions and 1 deletion.
  1. +6 −1 main.js
  2. +40 −0 tests/memory_leak.js
View
7 main.js
@@ -128,6 +128,10 @@ Spider.prototype.get = function (url, referer) {
request.get({url:url, headers:h, pool:self.pool}, function (e, resp, body) {
self.emit('log', debug, 'Response received for '+url+'.')
+ if (e) {
+ self.emit('log', error, e);
+ return;
+ }
if (resp.statusCode === 304) {
self.cache.get(url, function (c_) {
self._handler(url, referer, {fromCache:true, headers:c_.headers, body:c_.body})
@@ -191,7 +195,8 @@ Spider.prototype._handler = function (url, referer, response) {
} else {
r.fn.call(r, window, window.$);
}
- this.currentUrl = null;
+ this.currentUrl = null;
+ window.close(); //fix suggested by
}
}
Spider.prototype.log = function (level) {
View
40 tests/memory_leak.js
@@ -0,0 +1,40 @@
+/**
+ * AS taken from this gist https://gist.github.com/gmarcus/934787
+ * original dies after 840 fetches
+ * @type {*}
+ */
+
+var sys = require('sys');
+var util = require('util');
+var spider = require('../main');
+var counter = 0;
+
+spider()
+ .route('itunes.apple.com', '/us/genre/*', function (window, $) {
+ if (this.fromCache) return;
+
+ console.log("Fetching page: %s, for the %s th time", this.spider.currentUrl, ++counter);
+
+ // spider all genres
+ $('div#genre-nav.main.nav a').spider();
+
+ // spider all letters per genre
+ $('div#selectedgenre ul.list.alpha li a').spider();
+
+ // spider all numbered pages of letters per genre
+ $('div#selectedgenre ul.list.paginate li a').spider();
+
+
+ // // fetch apps JSON and store in a database (not implemented yet)
+ // $("#selectedgenre .column a").each(function(i,a) {
+ // // extract the iTunes URL
+ // var aHref = a.href;
+ // console.log("Recording " + aHref);
+ //
+ // });
+
+// console.log(util.inspect(this.spider, false, null));
+ })
+ .get('http://itunes.apple.com/us/genre/ios/id36?mt=8')
+ .log('info')
+;

0 comments on commit 1ccaa37

Please sign in to comment.