statbot

statbot makes it possible to gather statistics about urls

Installation

Install statbot from npm

npm install statbot

Usage

StatBot scripts should be standalone scripts, since these are forked into child workers.

var StatBot = require("statbot");

var options = {
    debug: false, // turn verbose mode off
    childProcesses: 10 // how many child forks to start
}

var stats = new StatBot(options);

// you only need to generate the link list while in Master mode
stats.on("links", function(callback){
    var links = ["http://link1", "http://link2"];
    callback(links);
});

stats.on("url", function(data){
    console.log(data);
    /*
     * data:
     *   url: http://www.link1
     *   meta:
     *       responseHeaders:
     *           date: ...
     *   error: String
     */
});

request.on("end", function(){
    console.log("Ready");
});

See examples dir for complete scripts

License

MIT