From f59e0889c9498d3555461227437c154b9a3ae792 Mon Sep 17 00:00:00 2001 From: Jiang Bian Date: Fri, 18 Oct 2013 02:17:23 -0500 Subject: [PATCH] d --- tweetf0rm/scheduler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tweetf0rm/scheduler.py b/tweetf0rm/scheduler.py index 6b9a0e1..48d3def 100644 --- a/tweetf0rm/scheduler.py +++ b/tweetf0rm/scheduler.py @@ -101,15 +101,17 @@ def enqueue(self, cmd): crawler_id = self.distribute_to() self.crawlers[crawler_id]['queue'][hash_cmd(cmd)] = cmd + self.crawlers[crawler_id]['crawler'].enqueue(cmd) if (self.verbose): logger.info("pusing [%s] to crawler: %s"%(hash_cmd(cmd), crawler_id)) def check_local_qsizes(self): + #logger.info(self.crawlers) return {crawler_id:len(self.crawlers[crawler_id]['queue']) for crawler_id in self.crawlers} - + def split(self, l, n): """ Yield successive n-sized chunks from l.""" for i in xrange(0, len(l), n):