From 8bb2ff8716548165ebbe476645b406f8c1179878 Mon Sep 17 00:00:00 2001 From: Jiang Bian Date: Tue, 12 Nov 2013 20:39:56 -0600 Subject: [PATCH] add support to clear a node's queues to get you a fresh start --- tweetf0rm/client.py | 6 ++++++ tweetf0rm/redis_helper.py | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/tweetf0rm/client.py b/tweetf0rm/client.py index 4941759..f2531cc 100644 --- a/tweetf0rm/client.py +++ b/tweetf0rm/client.py @@ -103,6 +103,8 @@ }, 'NODE_QSIZES': { + }, 'CLEAR_NODE_QUEUES': { + } } @@ -172,6 +174,8 @@ def cmd(config, args): #node_coordinator.remove_node(nid) #pp.pprint(node_coordinator.list_nodes()) raise NotImplemented("NotImplemented yet...") + elif (args.command == 'CLEAR_NODE_QUEUES'): + node_queue.clear_all_queues() else: args_dict = copy.copy(args.__dict__) cmd = new_cmd(args.command, args_dict) @@ -225,6 +229,8 @@ def print_avaliable_cmd(): '-nid/--node_id': dictionary['-nid/--node_id'] }, 'NODE_QSIZES':{ '-nid/--node_id': dictionary['-nid/--node_id'] + }, 'CLEAR_NODE_QUEUES':{ + '-nid/--node_id': dictionary['-nid/--node_id'] }} diff --git a/tweetf0rm/redis_helper.py b/tweetf0rm/redis_helper.py index 5f6489a..71506d3 100644 --- a/tweetf0rm/redis_helper.py +++ b/tweetf0rm/redis_helper.py @@ -92,6 +92,15 @@ class NodeQueue(RedisQueue): def __init__(self, node_id, redis_config=None): super(NodeQueue, self).__init__(node_id, redis_config=redis_config) + self.node_id = node_id + + def clear_all_queues(self): + '''This will not only clear the node queue (mostly for control cmds); but also the crawlers' cmd queues to give you a fresh start''' + #self.conn().delete('queue:%s*'%(self.node_id)) + for key in self.conn().keys('queue:%s:*'%self.node_id): + self.conn().delete(key) + + self.conn().delete('queue:%s'%self.node_id) class NodeCoordinator(RedisBase): '''