diff --git a/tweetf0rm/redis_helper.py b/tweetf0rm/redis_helper.py index 71506d3..b10b42a 100644 --- a/tweetf0rm/redis_helper.py +++ b/tweetf0rm/redis_helper.py @@ -124,7 +124,8 @@ def distribute_to_nodes(self, crawler_queue): qsizes = self.node_qsizes() - while (crawler_queue.get(timeout=60)): + cmd = crawler_queue.get(timeout=60) + while (cmd): node_id = get_keys_by_min_value(qsizes)[0] @@ -133,6 +134,8 @@ def distribute_to_nodes(self, crawler_queue): node.put(cmd) qsizes[node_id] += 1 + cmd = crawler_queue.get(timeout=60) + def clear(self): self.conn().delete('%s:*'%self.key) @@ -160,7 +163,7 @@ def node_qsizes(self): for crawler_queue_key in self.conn().keys('queue:%s:*'%node_id): qsize += self.conn().llen(crawler_queue_key) - qsizes[node_id] = node_qsize + qsizes[node_id] = qsize return qsizes