Permalink
Browse files

First working version with redis and mongo backends

  • Loading branch information...
0 parents commit 56fbcdc1a4adde691a5769d39485b5e4e885cda2 @bkeepers committed Sep 18, 2011
@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
19 Gemfile
@@ -0,0 +1,19 @@
+source "http://rubygems.org"
+gemspec :name => 'qu'
+
+group :mongo do
+ gemspec :name => 'qu-mongo', :development_group => :mongo
+end
+
+group :redis do
+ gemspec :name => 'qu-redis', :development_group => :redis
+end
+
+group :test do
+ gem 'SystemTimer', :platform => :mri_18
+ gem 'ruby-debug', :platform => :mri_18
+ gem 'rake'
+ gem 'rspec', '~> 2.0'
+ gem 'guard-rspec'
+ gem 'guard-bundler'
+end
@@ -0,0 +1,12 @@
+guard 'rspec', :version => 2 do
+ watch(%r{^spec/.+_spec\.rb$})
+ watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
+ watch(%r{^lib/qu/backend/spec\.rb$}) { |m| "spec/qu/backend" }
+ watch('spec/spec_helper.rb') { "spec" }
+ watch(%r{^spec/support/(.+)\.rb$}) { "spec" }
+end
+
+guard 'bundler' do
+ watch('Gemfile')
+ watch(/^.+\.gemspec/)
+end
@@ -0,0 +1,47 @@
+## Features
+
+* Multiple backends (redis, mongo, sql)
+* Resque-like API
+*
+
+## Installation
+
+ gem 'qu-redis'
+
+ Qu.configure do |c|
+ c.connection = Redis.new
+ c.fork = false
+ c.poll = 20 # for backends that need to poll instead of blocking
+ end
+
+## API
+
+ Qu.length('presentations')
+ Qu.work_off # work off all jobs until there are none
+ Qu.clear
+
+ class ProcessPresentation < Qu::Job.new(:presentation_id)
+ queue :mailers
+
+ def perform
+ presentation = Presentation.find(presentation_id)
+ # work here
+ end
+ end
+
+ job_id = Qu.enqueue ProcessPresentation, @presentation.id
+
+ ProcessPresentation.create(:presentation_id => 1)
+
+ Qu::Worker.new(*%w(presentations slides *)).start # or work_off
+
+## ToDo
+
+* worker.work
+* add job back on queue when worker dies
+* use queue specified in job class
+* configurable exception handling
+* callbacks (enqueue, process, error)
+* make poll timer configurable
+* logger
+* autoconfigure heroku connections
@@ -0,0 +1,24 @@
+require 'rspec/core/rake_task'
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:spec) do |t|
+ t.rspec_opts = %w[--color]
+ t.verbose = false
+end
+
+namespace :spec do
+ Backends = %w(mongo redis)
+
+ Backends.each do |backend|
+ desc "Run specs for #{backend} backend"
+ RSpec::Core::RakeTask.new(backend) do |t|
+ t.rspec_opts = %w[--color]
+ t.verbose = false
+ t.pattern = "spec/qu/backend/#{backend}_spec.rb"
+ end
+ end
+
+ task :backends => Backends
+end
+
+task :default => :spec
@@ -0,0 +1,4 @@
+require 'qu'
+require 'qu/backend/mongo'
+
+Qu.backend = Qu::Backend::Mongo.new
@@ -0,0 +1,4 @@
+require 'qu'
+require 'qu/backend/redis'
+
+Qu.backend = Qu::Backend::Redis.new
@@ -0,0 +1,22 @@
+require 'qu/version'
+require 'qu/job'
+require 'qu/backend/base'
+
+require 'forwardable'
+
+module Qu
+ autoload :Worker, 'qu/worker'
+
+ extend SingleForwardable
+ extend self
+
+ def_delegators :backend, :enqueue, :length, :queues, :reserve
+
+ def backend=(backend)
+ @backend = backend
+ end
+
+ def backend
+ @backend ||= Backend::Redis.new
+ end
+end
@@ -0,0 +1,18 @@
+require 'multi_json'
+
+module Qu
+ module Backend
+ class Base
+
+ private
+
+ def encode(data)
+ MultiJson.encode(data)
+ end
+
+ def decode(data)
+ MultiJson.decode(data)
+ end
+ end
+ end
+end
@@ -0,0 +1,68 @@
+require 'mongo'
+
+module Qu
+ module Backend
+ class Mongo < Base
+ def database
+ @database ||= ::Mongo::Connection.new.db('qu')
+ end
+
+ def clear(queue = queues)
+ Array(queue).each do |q|
+ jobs(q).drop
+ self[:queues].remove({:name => q})
+ end
+ end
+
+ def queues
+ self[:queues].find.map {|doc| doc['name'] }
+ end
+
+ def length(queue)
+ jobs(queue).count
+ end
+
+ def enqueue(klass, *args)
+ id = BSON::ObjectId.new
+ jobs(klass.queue).insert({:_id => id, :class => klass.to_s, :args => args})
+ self[:queues].update({:name => klass.queue}, {:name => klass.queue}, :upsert => true)
+ id
+ end
+
+ def reserve(worker, options = {:block => true})
+ worker.queues.each do |queue|
+ begin
+ doc = jobs(queue).find_and_modify(:remove => true)
+ return Job.load(doc['_id'], doc['class'], doc['args'])
+ rescue ::Mongo::OperationFailure
+ # No jobs in the queue
+ end
+ end
+
+ if options[:block]
+ sleep 5
+ retry
+ end
+ end
+
+ def release(job)
+
+ end
+
+ def delete(job)
+
+ end
+
+
+ private
+
+ def jobs(queue)
+ self["queue:#{queue}"]
+ end
+
+ def [](name)
+ database["qu.#{name}"]
+ end
+ end
+ end
+end
@@ -0,0 +1,58 @@
+require 'redis'
+
+module Qu
+ module Backend
+ class Redis < Base
+ def redis
+ @redis ||= ::Redis.connect
+ end
+
+ def enqueue(klass, *args)
+ data = encode('class' => klass.to_s, 'args' => args)
+ id = unique_id(data)
+ redis.set("job:#{id}", data)
+ redis.rpush("queue:#{klass.queue}", id)
+ redis.sadd('queues', klass.queue)
+ id
+ end
+
+ def length(queue)
+ redis.llen("queue:#{queue}")
+ end
+
+ def clear(queue = queues)
+ Array(queue).each do |q|
+ redis.srem('queues', q)
+ redis.del("queue:#{q}")
+ end
+ end
+
+ def queues
+ Array(redis.smembers('queues'))
+ end
+
+ def reserve(worker, options = {:block => true})
+ queues = worker.queues.map {|q| "queue:#{q}" }
+
+ if options[:block]
+ id = redis.blpop(*queues.push(0))[1]
+ else
+ queues.detect {|queue| id = redis.lpop(queue) }
+ end
+
+ if id
+ data = decode(redis.get("job:#{id}"))
+ redis.del("job:#{id}")
+ Job.load(id, data['class'], data['args'])
+ end
+ end
+
+ private
+
+ def unique_id(data)
+ Digest::MD5.hexdigest("#{Time.now.to_f} - #{rand} - #{data}")
+ end
+
+ end
+ end
+end
Oops, something went wrong.

0 comments on commit 56fbcdc

Please sign in to comment.