diff --git a/lib/queue.js b/lib/queue.js index 4a5089c..53b09c4 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -75,39 +75,46 @@ FetchQueue.prototype.exists = function(protocol,domain,port,path,callback) { if (!!this.scanIndex[url]) { callback(null,1); + return 1; } else { this.scanIndex[url] = true; callback(null,0); + return 1; } }; // Get last item in queue... FetchQueue.prototype.last = function(callback) { callback = callback && callback instanceof Function ? callback : function(){}; - var self = this; - - callback(null,self[self.length-1]); + var item, self = this; + item = self[self.length-1]; + callback(null, item); + return item; }; // Get item from queue FetchQueue.prototype.get = function(id, callback) { callback = callback && callback instanceof Function ? callback : function(){}; - var self = this; + var item, self = this; if (!isNaN(id) && self.length > id) { - return callback(null,self[id]); + item = self[id]; + callback(null, item); + return item; } }; // Get first unfetched item in the queue (and return its index) FetchQueue.prototype.oldestUnfetchedItem = function(callback) { callback = callback && callback instanceof Function ? callback : function(){}; - var self = this; + var item, self = this; for (var itemIndex = self.oldestUnfetchedIndex; itemIndex < self.length; itemIndex ++) { if (self[itemIndex].status === "queued") { self.oldestUnfetchedIndex = itemIndex; - return callback(null,self[itemIndex]); + item = self[itemIndex]; + callback(null, item); + return item; } } @@ -131,12 +138,13 @@ FetchQueue.prototype.max = function(statisticName,callback) { }); callback(null,maxStatisticValue); + return maxStatisticValue; }; // Gets the minimum total request time, request latency, or download time FetchQueue.prototype.min = function(statisticName,callback) { callback = callback && callback instanceof Function ? callback : function(){}; - var minStatisticValue = Infinity, self = this; + var minimum, minStatisticValue = Infinity, self = this; if (allowedStatistics.join().indexOf(statisticName) === -1) { // Not a recognised statistic! @@ -148,14 +156,15 @@ FetchQueue.prototype.min = function(statisticName,callback) { minStatisticValue = item.stateData[statisticName]; } }); - - callback(null,minStatisticValue === Infinity? 0 : minStatisticValue); + minimum = minStatisticValue === Infinity? 0 : minStatisticValue; + callback(null, minimum); + return minimum; }; // Gets the minimum total request time, request latency, or download time FetchQueue.prototype.avg = function(statisticName,callback) { callback = callback && callback instanceof Function ? callback : function(){}; - var NumberSum = 0, NumberCount = 0, self = this; + var average, NumberSum = 0, NumberCount = 0, self = this; if (allowedStatistics.join().indexOf(statisticName) === -1) { // Not a recognised statistic! @@ -168,8 +177,9 @@ FetchQueue.prototype.avg = function(statisticName,callback) { NumberCount ++; } }); - - callback(null,NumberSum / NumberCount); + average = NumberSum / NumberCount; + callback(null, average); + return average; }; // Gets the number of requests which have been completed. @@ -199,6 +209,7 @@ FetchQueue.prototype.countWithStatus = function(status,callback) { }); callback(null,queueItemsMatched); + return queueItemsMatched; }; // Gets the number of queue items with the given status @@ -214,18 +225,19 @@ FetchQueue.prototype.getWithStatus = function(status,callback) { }); callback(null,subqueue); + return subqueue; }; // Gets the number of requests which have failed for some reason FetchQueue.prototype.errors = function(callback) { callback = callback && callback instanceof Function ? callback : function(){}; - var self = this; + var total, failedCount, notFoundCount, self = this; - self.countWithStatus("failed",function(err1,failed) { - self.countWithStatus("notfound",function(err2,notfound) { - callback(null,failed + notfound); - }); - }); + failedCount = self.countWithStatus("failed"); + notFoundCount = self.countWithStatus("notfound"); + total = failedCount + notFoundCount; + callback(null, total); + return total; }; // Writes the queue to disk diff --git a/test/cookies.js b/test/cookies.js index 2a32d61..220786a 100644 --- a/test/cookies.js +++ b/test/cookies.js @@ -9,7 +9,7 @@ var cookies = [ "Set-Cookie: RMID=007f010019155170d6ca005f; Expires=Sat, 19 Apr 2014 05:31:54 GMT; Path=/; Domain=.nytimes.com;", "Set-cookie: adxcs=-; path=/; domain=.nytimes.com", "Set-Cookie: PREF=ID=8c63f2522e22574d:FF=0:TM=1366349569:LM=1366349569:S=p1Urbmfwfs-R573P; expires=Sun, 19-Apr-2015 05:32:49 GMT; path=/; domain=.google.com", - "Set-Cookie: NID=67=DhLO04YPAMlhETrTIe2oFPqWZfypQXLZfCIPItOvf70zhtUEMEItYfdVh6aROEzRHqtd9jHT6HJ7Oo93eqP3cjYNp8GgShfa6r0WVbsmQQRUvutbjBOPwzo7ybwYcWdB; expires=Sat, 19-Oct-2013 05:32:49 GMT; path=/; domain=.google.com; HttpOnly", + "Set-Cookie: NID=67=DhLO04YPAMlhETrTIe2oFPqWZfypQXLZfCIPItOvf70zhtUEMEItYfdVh6aROEzRHqtd9jHT6HJ7Oo93eqP3cjYNp8GgShfa6r0WVbsmQQRUvutbjBOPwzo7ybwYcWdB; expires=Sat, 19-Oct-2015 05:32:49 GMT; path=/; domain=.google.com; HttpOnly", "Set-Cookie: fpc=d=Yq1z8hbA9WextmPFlb7suMTfMRgtSc2FyzAB7now1ExfUZ.eW7s4QSwSKlB6ZB0juN8OLZxWf_XXEIcspYaQmVVD0mD0xJ.xpXBCSw5Dl_Ql6n.RLoM.7CnTbNSsiSr2fkNiCN47tRUB4j8iWevNwQdFDn1hB8z8t1xwWt76n.sLIRY9p2_jTBhukfSD4SBpBkJhI1o-&v=2; expires=Sat, 19-Apr-2014 05:48:42 GMT; path=/; domain=www.yahoo.com", "Set-Cookie: test=test; path=/test; domain=test.com" ]; diff --git a/test/jshint.js b/test/jshint.js index 4ade490..b6a8bef 100644 --- a/test/jshint.js +++ b/test/jshint.js @@ -25,6 +25,9 @@ describe("Core code",function() { it("module `" + item + "` should pass JSHint with no errors",function() { + var slowThresholdMilliseconds = 200; + this.slow(slowThresholdMilliseconds); + JSHINT(code,{ "indent": 4, "undef": true diff --git a/test/reliability.js b/test/reliability.js index 6d50366..bcf2969 100644 --- a/test/reliability.js +++ b/test/reliability.js @@ -12,6 +12,8 @@ describe("Crawler reliability",function() { it("should be able to handle a timeout",function(done) { + this.slow('1s') + var localCrawler = Crawler.crawl("http://127.0.0.1:3000/timeout"); localCrawler.timeout = 200; diff --git a/test/testcrawl.js b/test/testcrawl.js index 3ce3305..928fc8b 100644 --- a/test/testcrawl.js +++ b/test/testcrawl.js @@ -46,6 +46,8 @@ describe("Test Crawl",function() { it("should support async event listeners for manual discovery",function(done) { + this.slow('1s') + // Use a different crawler this time asyncCrawler.discoverResources = false; asyncCrawler.queueURL("http://127.0.0.1:3000/async-stage1");