From 5bebd9af26c283b7bec2799f99b066b423728657 Mon Sep 17 00:00:00 2001 From: Francesco Laurita Date: Sun, 29 Jun 2014 09:57:37 -0700 Subject: [PATCH] example on how to use overflow management --- examples/basic.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/basic.rb b/examples/basic.rb index 1e41a81..1e2ab2d 100644 --- a/examples/basic.rb +++ b/examples/basic.rb @@ -20,7 +20,14 @@ user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71', # Use 5 threads workers: 5, - # Logs goes to the crawler.log file + # Queue overflow settings: + # * No more than 5000 elements on the Redis queue + # * Exceeded Items will stored on Mongo into 'rubygems_queue_overflow' collection + # * Check cycle is done every 60 sec + queue_items_limit: 5_000, + queue_overflow_adapter: Polipus::QueueOverflow.mongo_queue(mongo, 'rubygems_queue_overflow'), + queue_overflow_manager_check_time: 60, + # Logs goes to the stdout logger: Logger.new(STDOUT) } Polipus::Plugin.register Polipus::Plugin::Cleaner, reset: true