Permalink
Browse files

huntsman now spiders many (not all) GETs

  • Loading branch information...
1 parent ce9841b commit 48f67205130c1d044c34be194b9402584b15ba2e @cainus committed Nov 2, 2012
Showing with 17 additions and 13 deletions.
  1. +17 −3 huntsman.js
  2. +0 −9 test/test_fixtures/resources/happy.js
  3. +0 −1 test/test_fixtures/resources/index.js
View
@@ -25,31 +25,35 @@ Huntsman.prototype.capture = function(url, method, headers, body, cb){
if (this.debug){
console.log('<-- ', method, headers, body);
}
+ this.beenThere.push(method + "|" + url);
hottap(url).request(method, headers, body, function(err, response){
if (err){
that.emit('error', err);
if (that.stopOnError){
that.shouldStop = true;
}
+ cb(err);
} else {
if (that.debug){
console.log('--> ', response);
}
that.emit('response', response);
if (response.headers['content-type'] === 'application/json'){
console.log("harvesting");
- that.harvest(response.body);
+ that.harvest(response.body, function(){
+ cb(response);
+ });
} else {
if (that.debug){
console.log('--> (non-json response)');
}
+ cb('non-json response', response);
}
- cb(err, response);
}
});
};
-Huntsman.prototype.harvest = function(payload){
+Huntsman.prototype.harvest = function(payload, cb){
var that = this;
var obj = {};
console.log("payload: ", payload);
@@ -69,10 +73,19 @@ Huntsman.prototype.harvest = function(payload){
that.emit('link', rel, link);
if (!!that.autoGET && !that.shouldStop){
console.log("about to follow... ", link);
+ if (!that.hasBeenTo('GET', link.href)){
+ that.capture(link.href, 'GET', {}, '', function(){
+ cb();
+ });
+ }
}
});
};
+Huntsman.prototype.hasBeenTo = function(method, url){
+ return (this.beenThere.indexOf(method + '|' + url) !== -1);
+};
+
Huntsman.prototype.hunt = function(url){
var that = this;
this.root = url || this.root;
@@ -101,6 +114,7 @@ hunter.on('response', function(response){
});
hunter.on('end', function(){
+ console.log(hunter.beenThere);
console.log('done');
});
@@ -1,9 +0,0 @@
-exports.handler = {
-
- GET : function($){
- $.res.setHeader('Content-type','application/json');
- $.res.end(JSON.stringify($.uri.help()));
- }
-
-
-};
@@ -5,7 +5,6 @@ exports.handler = {
GET : function($){
$.json({})
.link('artists', $.uri.child('artist'))
- .link('happy', $.uri.child('happy'))
.link('many', $.uri.child('many'))
.link('qstring', $.uri.child('qstring'))
.link('restricted', $.uri.child('restricted'))

0 comments on commit 48f6720

Please sign in to comment.