A queue that refreshes itself when it gets empty or stale, so you can keep popping

Install

gem install refilling_queue

Usage

queue = RefillingQueue.new redis_client, "my_queue", refresh_every: 30.seconds do
  expensive_operation.map(&:id)
end

begin
  queue.pop
rescue RefillingQueue::Locked
  # queue was empty, refilling failed because other process is already trying it
end

queue.pop -> return id
... # queue empty ?
queue.pop -> run block -> store new ids -> return id
... # 30 seconds elapsed (global expires_at stored in reque_client) ?
queue.pop -> run block -> store new ids -> return id
...
queue.pop -> run block -> empty result -> return nil

Pagination

queue = RefillingQueue.new redis_client, "my_queue", refresh_every: 30.seconds, paginate: true do |page|
  expensive_operation(:page => page).map(&:id)
end

Author

Michael Grosser
michael@grosser.it
License: MIT