diff --git a/tweetf0rm/bootstrap.py b/tweetf0rm/bootstrap.py index ae2ac68..050929b 100644 --- a/tweetf0rm/bootstrap.py +++ b/tweetf0rm/bootstrap.py @@ -139,7 +139,7 @@ def start_server(config, proxies): # block, the main process...for a command if(not scheduler.is_alive()): - logger.info("no crawler is alive... i'm done too...") + logger.info("no crawler is alive... waiting to recreate all crawlers...") time.sleep(120) # sleep for a minute and retry continue diff --git a/tweetf0rm/scheduler.py b/tweetf0rm/scheduler.py index 35fa1de..3c00627 100644 --- a/tweetf0rm/scheduler.py +++ b/tweetf0rm/scheduler.py @@ -81,7 +81,7 @@ def new_crawler(self, node_id, apikeys, config, crawler_proxies = None): crawler = UserRelationshipCrawler(node_id, crawler_id, copy.copy(apikeys), handlers=[create_handler(file_handler_config)], redis_config=copy.copy(config['redis_config']), proxies=crawler_proxies) if (crawler_id in self.crawlers): - self.crawlers[crawler_id].clear() + #self.crawlers[crawler_id].clear() del self.crawlers[crawler_id] self.crawlers[crawler_id] = {