Permalink
Browse files

Fixed two bugs. In one, the node queue size used an incorrect variable

name. In the other, the command to redistribute to other nodes wasn't
being populated.
  • Loading branch information...
1 parent 6c87fb1 commit 336589045663c40a7cdfb16b8d345ab5238226cc Cody Buntain committed Apr 3, 2014
Showing with 5 additions and 2 deletions.
  1. +5 −2 tweetf0rm/redis_helper.py
@@ -124,7 +124,8 @@ def distribute_to_nodes(self, crawler_queue):
qsizes = self.node_qsizes()
- while (crawler_queue.get(timeout=60)):
+ cmd = crawler_queue.get(timeout=60)
+ while (cmd):
node_id = get_keys_by_min_value(qsizes)[0]
@@ -133,6 +134,8 @@ def distribute_to_nodes(self, crawler_queue):
node.put(cmd)
qsizes[node_id] += 1
+ cmd = crawler_queue.get(timeout=60)
+
def clear(self):
self.conn().delete('%s:*'%self.key)
@@ -160,7 +163,7 @@ def node_qsizes(self):
for crawler_queue_key in self.conn().keys('queue:%s:*'%node_id):
qsize += self.conn().llen(crawler_queue_key)
- qsizes[node_id] = node_qsize
+ qsizes[node_id] = qsize
return qsizes

0 comments on commit 3365890

Please sign in to comment.