diff --git a/tweetf0rm/scheduler.py b/tweetf0rm/scheduler.py index 6b9a0e1..48d3def 100644 --- a/tweetf0rm/scheduler.py +++ b/tweetf0rm/scheduler.py @@ -101,15 +101,17 @@ def enqueue(self, cmd): crawler_id = self.distribute_to() self.crawlers[crawler_id]['queue'][hash_cmd(cmd)] = cmd + self.crawlers[crawler_id]['crawler'].enqueue(cmd) if (self.verbose): logger.info("pusing [%s] to crawler: %s"%(hash_cmd(cmd), crawler_id)) def check_local_qsizes(self): + #logger.info(self.crawlers) return {crawler_id:len(self.crawlers[crawler_id]['queue']) for crawler_id in self.crawlers} - + def split(self, l, n): """ Yield successive n-sized chunks from l.""" for i in xrange(0, len(l), n):