From 336589045663c40a7cdfb16b8d345ab5238226cc Mon Sep 17 00:00:00 2001 From: Cody Buntain Date: Thu, 3 Apr 2014 15:38:21 -0400 Subject: [PATCH] Fixed two bugs. In one, the node queue size used an incorrect variable name. In the other, the command to redistribute to other nodes wasn't being populated. --- tweetf0rm/redis_helper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tweetf0rm/redis_helper.py b/tweetf0rm/redis_helper.py index 71506d3..b10b42a 100644 --- a/tweetf0rm/redis_helper.py +++ b/tweetf0rm/redis_helper.py @@ -124,7 +124,8 @@ def distribute_to_nodes(self, crawler_queue): qsizes = self.node_qsizes() - while (crawler_queue.get(timeout=60)): + cmd = crawler_queue.get(timeout=60) + while (cmd): node_id = get_keys_by_min_value(qsizes)[0] @@ -133,6 +134,8 @@ def distribute_to_nodes(self, crawler_queue): node.put(cmd) qsizes[node_id] += 1 + cmd = crawler_queue.get(timeout=60) + def clear(self): self.conn().delete('%s:*'%self.key) @@ -160,7 +163,7 @@ def node_qsizes(self): for crawler_queue_key in self.conn().keys('queue:%s:*'%node_id): qsize += self.conn().llen(crawler_queue_key) - qsizes[node_id] = node_qsize + qsizes[node_id] = qsize return qsizes