From 0a39370d53cb8178c365bb8e1ecb7864399521c5 Mon Sep 17 00:00:00 2001 From: Jade Rubick Date: Sun, 29 May 2011 07:02:20 -0700 Subject: [PATCH 01/12] Added w.dir to set the current directory. --- examples/god/resque.god | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/god/resque.god b/examples/god/resque.god index d28c90088..a2eaf0c11 100644 --- a/examples/god/resque.god +++ b/examples/god/resque.god @@ -4,6 +4,7 @@ num_workers = rails_env == 'production' ? 5 : 2 num_workers.times do |num| God.watch do |w| + w.dir = "#{rails_root}" w.name = "resque-#{num}" w.group = 'resque' w.interval = 30.seconds From d1cfe4dd495be47f44d6d7ef090212eae854cbf3 Mon Sep 17 00:00:00 2001 From: Matthijs van der Vleuten Date: Sun, 29 May 2011 12:53:01 +0200 Subject: [PATCH 02/12] Shorten the backtrace logged by Resque::Failure::Redis to only include the job by removing Resque and its callers (like Rake) from the backtrace. --- lib/resque/failure/redis.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/resque/failure/redis.rb b/lib/resque/failure/redis.rb index fc060764e..68feca64b 100644 --- a/lib/resque/failure/redis.rb +++ b/lib/resque/failure/redis.rb @@ -9,7 +9,7 @@ def save :payload => payload, :exception => exception.class.to_s, :error => exception.to_s, - :backtrace => Array(exception.backtrace), + :backtrace => filter_backtrace(Array(exception.backtrace)), :worker => worker.to_s, :queue => queue } @@ -41,6 +41,10 @@ def self.remove(index) Resque.redis.lset(:failed, index, id) Resque.redis.lrem(:failed, 1, id) end + + def filter_backtrace(backtrace) + backtrace.first(backtrace.index {|item| item.include?('/lib/resque/job.rb')}) + end end end end From 285fef51d3a1d62b19ccbda43462ff0f1a49814f Mon Sep 17 00:00:00 2001 From: Josh Kalderimis Date: Mon, 20 Jun 2011 15:53:43 +0200 Subject: [PATCH 03/12] instead of using the json gem, or trying to require yajl, lets use multi json which takes care of this for us and includes a vendored json implementation if one is not available --- lib/resque.rb | 6 ------ lib/resque/helpers.rb | 24 +++++++----------------- resque.gemspec | 2 +- 3 files changed, 8 insertions(+), 24 deletions(-) diff --git a/lib/resque.rb b/lib/resque.rb index 20fb7e4f1..4b6e1dc0e 100644 --- a/lib/resque.rb +++ b/lib/resque.rb @@ -1,11 +1,5 @@ require 'redis/namespace' -begin - require 'yajl' -rescue LoadError - require 'json' -end - require 'resque/version' require 'resque/errors' diff --git a/lib/resque/helpers.rb b/lib/resque/helpers.rb index 3ac57eeec..8150b5876 100644 --- a/lib/resque/helpers.rb +++ b/lib/resque/helpers.rb @@ -1,3 +1,5 @@ +require 'multi_json' + module Resque # Methods used by various classes in Resque. module Helpers @@ -11,29 +13,17 @@ def redis # Given a Ruby object, returns a string suitable for storage in a # queue. def encode(object) - if defined? Yajl - Yajl::Encoder.encode(object) - else - object.to_json - end + ::MultiJson.encode(object) end # Given a string, returns a Ruby object. def decode(object) return unless object - if defined? Yajl - begin - Yajl::Parser.parse(object, :check_utf8 => false) - rescue Yajl::ParseError => e - raise DecodeException, e - end - else - begin - JSON.parse(object) - rescue JSON::ParserError => e - raise DecodeException, e - end + begin + ::MultiJson.decode(object) + rescue ::MultiJson::DecodeError => e + raise DecodeException, e end end diff --git a/resque.gemspec b/resque.gemspec index 8c4df9651..a3c913b9e 100644 --- a/resque.gemspec +++ b/resque.gemspec @@ -24,7 +24,7 @@ Gem::Specification.new do |s| s.add_dependency "redis-namespace", "~> 1.0.2" s.add_dependency "vegas", "~> 0.1.2" s.add_dependency "sinatra", ">= 0.9.2" - s.add_dependency "json", ">= 1.4.6", "< 1.6" + s.add_dependency "multi_json", "~> 1.0" s.description = < Date: Wed, 13 Jul 2011 19:25:07 -0700 Subject: [PATCH 04/12] bundle install --- README.markdown | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/README.markdown b/README.markdown index 6bea746a0..f52097b34 100644 --- a/README.markdown +++ b/README.markdown @@ -560,13 +560,8 @@ together. But, it's not that hard. Resque Dependencies ------------------- - gem install redis redis-namespace yajl-ruby vegas sinatra - -If you cannot install `yajl-ruby` (JRuby?), you can install the `json` -gem and Resque will use it instead. - -When problems arise, make sure you have the newest versions of the -`redis` and `redis-namespace` gems. + $ gem install bundler + $ bundle install Installing Resque From 2764529ae6ba72fd5b762a35dc93aa82c7f6ddd3 Mon Sep 17 00:00:00 2001 From: Florian Frank Date: Wed, 6 Jul 2011 15:12:35 +0200 Subject: [PATCH 05/12] Fix bug causing working to always return the max Apparently value isn't (always?) nil, but an empty hash, if the worker isn't working. --- lib/resque/worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resque/worker.rb b/lib/resque/worker.rb index b5f6e6529..5a12f0d8a 100644 --- a/lib/resque/worker.rb +++ b/lib/resque/worker.rb @@ -36,7 +36,7 @@ def self.working names.map! { |name| "worker:#{name}" } reportedly_working = redis.mapped_mget(*names).reject do |key, value| - value.nil? + value.nil? || value.empty? end reportedly_working.keys.map do |key| find key.sub("worker:", '') From 1f1ff49b80ee9119f11af82f2708d965885edcc5 Mon Sep 17 00:00:00 2001 From: Matt Duncan Date: Wed, 6 Jul 2011 17:56:17 -0400 Subject: [PATCH 06/12] Adding install instructions for Rails 3 --- README.markdown | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index f52097b34..66407daa5 100644 --- a/README.markdown +++ b/README.markdown @@ -601,7 +601,7 @@ Alternately you can define a `resque:setup` hook in your Rakefile if you don't want to load your app every time rake runs. -### In a Rails app, as a gem +### In a Rails 2.x app, as a gem First install the gem. @@ -632,7 +632,7 @@ Don't forget you can define a `resque:setup` hook in `lib/tasks/whatever.rake` that loads the `environment` task every time. -### In a Rails app, as a plugin +### In a Rails 2.x app, as a plugin $ ./script/plugin install git://github.com/defunkt/resque @@ -647,6 +647,40 @@ Don't forget you can define a `resque:setup` hook in `lib/tasks/whatever.rake` that loads the `environment` task every time. +### In a Rails 3 app, as a gem + +First include it in your Gemfile. + + $ cat Gemfile + ... + gem 'resque' + ... + +Next install it with Bundler. + + $ bundle install + +Now start your application: + + $ rails server + +That's it! You can now create Resque jobs from within your app. + +To start a worker, add this to a file in `lib/tasks` (ex: +`lib/tasks/resque.rake`): + +``` ruby +require 'resque/tasks' +``` + +Now: + + $ QUEUE=* rake environment resque:work + +Don't forget you can define a `resque:setup` hook in +`lib/tasks/whatever.rake` that loads the `environment` task every time. + + Configuration ------------- From 7b1ceea442a9638bee752a012006f849e2cc3162 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Wed, 13 Jul 2011 19:44:09 -0700 Subject: [PATCH 07/12] fix failure introduced by d1cfe4dd495be47f44d6d7ef090212eae854cbf3 --- lib/resque/failure/redis.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/resque/failure/redis.rb b/lib/resque/failure/redis.rb index 68feca64b..8ebee78c1 100644 --- a/lib/resque/failure/redis.rb +++ b/lib/resque/failure/redis.rb @@ -43,7 +43,8 @@ def self.remove(index) end def filter_backtrace(backtrace) - backtrace.first(backtrace.index {|item| item.include?('/lib/resque/job.rb')}) + index = backtrace.index { |item| item.include?('/lib/resque/job.rb') } + backtrace.first(index.to_i) end end end From 40f8b45a4c99110ca9e78fabb4d1006f692a6ba0 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Wed, 13 Jul 2011 19:44:20 -0700 Subject: [PATCH 08/12] fix failure introduced by 285fef51d3a1d62b19ccbda43462ff0f1a49814f --- test/worker_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/worker_test.rb b/test/worker_test.rb index e62de3337..fe5410905 100644 --- a/test/worker_test.rb +++ b/test/worker_test.rb @@ -33,7 +33,7 @@ end test "fails uncompleted jobs on exit" do - job = Resque::Job.new(:jobs, [GoodJob, "blah"]) + job = Resque::Job.new('jobs', ['GoodJob', "blah"]) @worker.working_on(job) @worker.unregister_worker assert_equal 1, Resque::Failure.count From aa24424c9484858c51194eb903f30c68fa0c88cb Mon Sep 17 00:00:00 2001 From: Josh Huckabee Date: Fri, 10 Jun 2011 09:54:59 -0700 Subject: [PATCH 09/12] Added a before_enqueue hook. --- docs/HOOKS.md | 3 +++ lib/resque.rb | 6 ++++++ lib/resque/plugin.rb | 5 +++++ test/job_hooks_test.rb | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/docs/HOOKS.md b/docs/HOOKS.md index c091a60e3..c6a5f66d9 100644 --- a/docs/HOOKS.md +++ b/docs/HOOKS.md @@ -66,6 +66,9 @@ An unnamed hook (`before_perform`) will be executed first. The available hooks are: +* `before _enqueue`: Called with the job args before a job is placed on the queue. + If the hook returns `false`, the job will not be placed on the queue. + * `after_enqueue`: Called with the job args after a job is placed on the queue. Any exception raised propagates up to the code which queued the job. diff --git a/lib/resque.rb b/lib/resque.rb index 4b6e1dc0e..a3ec9416b 100644 --- a/lib/resque.rb +++ b/lib/resque.rb @@ -224,6 +224,12 @@ def watch_queue(queue) # # This method is considered part of the `stable` API. def enqueue(klass, *args) + # Perform before_enqueue hooks. Don't perform enqueue if any hook returns false + before_hooks = Plugin.before_enqueue_hooks(klass).collect do |hook| + klass.send(hook, *args) + end + return if before_hooks.any? { |result| result == false } + Job.create(queue_from_class(klass), klass, *args) Plugin.after_enqueue_hooks(klass).each do |hook| diff --git a/lib/resque/plugin.rb b/lib/resque/plugin.rb index 3152dd4bc..5f9c598c5 100644 --- a/lib/resque/plugin.rb +++ b/lib/resque/plugin.rb @@ -47,5 +47,10 @@ def failure_hooks(job) def after_enqueue_hooks(job) job.methods.grep(/^after_enqueue/).sort end + + # Given an object, returns a list `before_enqueue` hook names. + def before_enqueue_hooks(job) + job.methods.grep(/^before_enqueue/).sort + end end end diff --git a/test/job_hooks_test.rb b/test/job_hooks_test.rb index 90a13a9ae..356ee32a2 100644 --- a/test/job_hooks_test.rb +++ b/test/job_hooks_test.rb @@ -250,6 +250,46 @@ def self.perform(history) end end + +context "Resque::Job before_enqueue" do + include PerformJob + + class ::BeforeEnqueueJob + @queue = :jobs + def self.before_enqueue_record_history(history) + history << :before_enqueue + end + + def self.perform(history) + end + end + + class ::BeforeEnqueueJobAbort + @queue = :jobs + def self.before_enqueue_abort(history) + false + end + + def self.perform(history) + end + end + + test "the before enqueue hook should run" do + history = [] + @worker = Resque::Worker.new(:jobs) + Resque.enqueue(BeforeEnqueueJob, history) + @worker.work(0) + assert_equal history, [:before_enqueue], "before_enqueue was not run" + end + + test "a before enqueue hook that returns false should prevent the job from getting queued" do + history = [] + @worker = Resque::Worker.new(:jobs) + Resque.enqueue(BeforeEnqueueJobAbort, history) + assert_equal 0, Resque.size(:jobs) + end +end + context "Resque::Job all hooks" do include PerformJob From 9255bf9052222acc05900c2a41f929274b39c3cf Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Wed, 13 Jul 2011 20:30:06 -0700 Subject: [PATCH 10/12] Revert "fix failure introduced by 285fef51d3a1d62b19ccbda43462ff0f1a49814f" This reverts commit 40f8b45a4c99110ca9e78fabb4d1006f692a6ba0. --- test/worker_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/worker_test.rb b/test/worker_test.rb index fe5410905..e62de3337 100644 --- a/test/worker_test.rb +++ b/test/worker_test.rb @@ -33,7 +33,7 @@ end test "fails uncompleted jobs on exit" do - job = Resque::Job.new('jobs', ['GoodJob', "blah"]) + job = Resque::Job.new(:jobs, [GoodJob, "blah"]) @worker.working_on(job) @worker.unregister_worker assert_equal 1, Resque::Failure.count From bdf4f5ff030d67fca16e611d2de850c288c00429 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Wed, 13 Jul 2011 20:47:28 -0700 Subject: [PATCH 11/12] use yajl and json in test group --- Gemfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gemfile b/Gemfile index 9b421313b..f759d235b 100644 --- a/Gemfile +++ b/Gemfile @@ -7,4 +7,6 @@ group :test do gem "rack-test", "~> 0.5" gem "mocha", "~> 0.9.7" gem "leftright", :platforms => :mri_18 + gem "yajl-ruby", "~>0.8.2", :platforms => :mri + gem "json", "~>1.5.3", :platforms => :jruby end From d85097dc66efce1f58621adb0779bbd4989db54d Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Wed, 13 Jul 2011 21:23:34 -0700 Subject: [PATCH 12/12] OkJson won't work, raise if it's in use. --- lib/resque/helpers.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/resque/helpers.rb b/lib/resque/helpers.rb index 8150b5876..1d045aa07 100644 --- a/lib/resque/helpers.rb +++ b/lib/resque/helpers.rb @@ -1,5 +1,11 @@ require 'multi_json' +# OkJson won't work because it doesn't serialize symbols +# in the same way yajl and json do. +if MultiJson.engine.to_s == 'MultiJson::Engines::OkJson' + raise "Please install the yajl-ruby or json gem" +end + module Resque # Methods used by various classes in Resque. module Helpers