Skip to content
This repository
Browse code

remove excess whitespace

  • Loading branch information...
commit 0581928d6a38dc0835092045ce40b25435a5246a 1 parent e9499f5
risk danger olson technoweenie authored
159 lib/delayed/job.rb
... ... @@ -1,92 +1,89 @@
1 1 module Delayed
2 2
3 3 class DeserializationError < StandardError
4   - end
  4 + end
5 5
6   - class Job < ActiveRecord::Base
  6 + class Job < ActiveRecord::Base
7 7 set_table_name :delayed_jobs
8 8
9 9 cattr_accessor :worker_name
10 10 self.worker_name = "pid:#{Process.pid}"
11   -
12   -
  11 +
13 12 NextTaskSQL = '`run_at` <= ? AND (`locked_at` IS NULL OR `locked_at` < ?) OR (`locked_by` = ?)'
14 13 NextTaskOrder = 'priority DESC, run_at ASC'
15 14 ParseObjectFromYaml = /\!ruby\/\w+\:([^\s]+)/
16 15
17 16 class LockError < StandardError
18   - end
  17 + end
19 18
20 19 def self.clear_locks!
21 20 connection.execute "UPDATE #{table_name} SET `locked_by`=NULL, `locked_at`=NULL WHERE `locked_by`=#{quote_value(worker_name)}"
22 21 end
23   -
  22 +
24 23 def payload_object
25 24 @payload_object ||= deserialize(self['handler'])
26 25 end
27   -
  26 +
28 27 def payload_object=(object)
29 28 self['handler'] = object.to_yaml
30 29 end
31   -
32   - def reshedule(message, time = nil)
  30 +
  31 + def reshedule(message, time = nil)
33 32 time ||= Job.db_time_now + (attempts ** 4).seconds + 5
34   -
  33 +
35 34 self.attempts += 1
36 35 self.run_at = time
37   - self.last_error = message
  36 + self.last_error = message
38 37 self.unlock
39 38 save!
40   - end
41   -
42   -
  39 + end
  40 +
43 41 def self.enqueue(object, priority = 0)
44 42 unless object.respond_to?(:perform)
45   - raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
  43 + raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
46 44 end
47   -
48   - Job.create(:payload_object => object, :priority => priority)
49   - end
50   -
  45 +
  46 + Job.create(:payload_object => object, :priority => priority)
  47 + end
  48 +
51 49 def self.find_available(limit = 5)
52 50 time_now = db_time_now
53   - ActiveRecord::Base.silence do
  51 + ActiveRecord::Base.silence do
54 52 find(:all, :conditions => [NextTaskSQL, time_now, time_now, worker_name], :order => NextTaskOrder, :limit => limit)
55 53 end
56 54 end
57   -
  55 +
58 56 # Get the payload of the next job we can get an exclusive lock on.
59 57 # If no jobs are left we return nil
60   - def self.reserve(max_run_time = 4.hours)
61   -
62   - # We get up to 5 jobs from the db. In face we cannot get exclusive access to a job we try the next.
63   - # this leads to a more even distribution of jobs across the worker processes
64   - find_available(5).each do |job|
  58 + def self.reserve(max_run_time = 4.hours)
  59 +
  60 + # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
  61 + # This leads to a more even distribution of jobs across the worker processes
  62 + find_available(5).each do |job|
65 63 begin
66 64 job.lock_exclusively!(max_run_time, worker_name)
67   - yield job.payload_object
  65 + yield job.payload_object
68 66 job.destroy
69   - return job
  67 + return job
70 68 rescue LockError
71 69 # We did not get the lock, some other worker process must have
72 70 puts "failed to aquire exclusive lock for #{job.id}"
73   - rescue StandardError => e
74   - job.reshedule e.message
75   - return job
  71 + rescue StandardError => e
  72 + job.reshedule e.message
  73 + return job
76 74 end
77 75 end
78 76
79 77 nil
80   - end
  78 + end
81 79
82 80 # This method is used internally by reserve method to ensure exclusive access
83 81 # to the given job. It will rise a LockError if it cannot get this lock.
84 82 def lock_exclusively!(max_run_time, worker = worker_name)
85   - now = self.class.db_time_now
86   -
87   - affected_rows = if locked_by != worker
88   -
89   -
  83 + now = self.class.db_time_now
  84 +
  85 + affected_rows = if locked_by != worker
  86 +
90 87 # We don't own this job so we will update the locked_by name and the locked_at
91 88 connection.update(<<-end_sql, "#{self.class.name} Update to aquire exclusive lock")
92 89 UPDATE #{self.class.table_name}
@@ -94,9 +91,9 @@ def lock_exclusively!(max_run_time, worker = worker_name)
94 91 WHERE #{self.class.primary_key} = #{quote_value(id)} AND (`locked_at` IS NULL OR `locked_at` < #{quote_value(now + max_run_time)})
95 92 end_sql
96 93
97   - else
98   -
99   - # We alrady own this job, this may happen if the job queue crashes.
  94 + else
  95 +
  96 + # We already own this job, this may happen if the job queue crashes.
100 97 # Simply resume and update the locked_at
101 98 connection.update(<<-end_sql, "#{self.class.name} Update exclusive lock")
102 99 UPDATE #{self.class.table_name}
@@ -105,95 +102,95 @@ def lock_exclusively!(max_run_time, worker = worker_name)
105 102 end_sql
106 103
107 104 end
108   -
  105 +
109 106 unless affected_rows == 1
110 107 raise LockError, "Attempted to aquire exclusive lock failed"
111   - end
112   -
  108 + end
  109 +
113 110 self.locked_at = now
114   - self.locked_by = worker
115   - end
116   -
  111 + self.locked_by = worker
  112 + end
  113 +
117 114 def unlock
118 115 self.locked_at = nil
119 116 self.locked_by = nil
120 117 end
121   -
  118 +
122 119 def self.work_off(num = 100)
123 120 success, failure = 0, 0
124   -
  121 +
125 122 num.times do
126   -
  123 +
127 124 job = self.reserve do |j|
128 125 begin
129   - j.perform
  126 + j.perform
130 127 success += 1
131   - rescue
  128 + rescue
132 129 failure += 1
133 130 raise
134 131 end
135 132 end
136   -
  133 +
137 134 break if job.nil?
138   - end
139   -
  135 + end
  136 +
140 137 return [success, failure]
141 138 end
142   -
  139 +
143 140 private
144   -
145   - def deserialize(source)
  141 +
  142 + def deserialize(source)
146 143 attempt_to_load_file = true
147   -
148   - begin
149   - handler = YAML.load(source) rescue nil
150   - return handler if handler.respond_to?(:perform)
151   -
  144 +
  145 + begin
  146 + handler = YAML.load(source) rescue nil
  147 + return handler if handler.respond_to?(:perform)
  148 +
152 149 if handler.nil?
153 150 if source =~ ParseObjectFromYaml
154   -
  151 +
155 152 # Constantize the object so that ActiveSupport can attempt
156 153 # its auto loading magic. Will raise LoadError if not successful.
157 154 attempt_to_load($1)
158   -
  155 +
159 156 # If successful, retry the yaml.load
160 157 handler = YAML.load(source)
161   - return handler if handler.respond_to?(:perform)
  158 + return handler if handler.respond_to?(:perform)
162 159 end
163 160 end
164   -
  161 +
165 162 if handler.is_a?(YAML::Object)
166   -
  163 +
167 164 # Constantize the object so that ActiveSupport can attempt
168 165 # its auto loading magic. Will raise LoadError if not successful.
169 166 attempt_to_load(handler.class)
170   -
  167 +
171 168 # If successful, retry the yaml.load
172 169 handler = YAML.load(source)
173 170 return handler if handler.respond_to?(:perform)
174 171 end
175   -
176   - raise DeserializationError, 'Job failed to load: Unknown handler. Try to manually require the appropiate file.'
177   -
  172 +
  173 + raise DeserializationError, 'Job failed to load: Unknown handler. Try to manually require the appropiate file.'
  174 +
178 175 rescue TypeError, LoadError, NameError => e
179   -
180   - raise DeserializationError, "Job failed to load: #{e.message}. Try to manually require the required file."
  176 +
  177 + raise DeserializationError, "Job failed to load: #{e.message}. Try to manually require the required file."
181 178 end
182 179 end
183   -
  180 +
184 181 def attempt_to_load(klass)
185   - klass.constantize
  182 + klass.constantize
186 183 end
187 184
188 185 def self.db_time_now
189   - (ActiveRecord::Base.default_timezone == :utc) ? Time.now.utc : Time.now
  186 + (ActiveRecord::Base.default_timezone == :utc) ? Time.now.utc : Time.now
190 187 end
191   -
192   - protected
193   -
  188 +
  189 + protected
  190 +
194 191 def before_save
195 192 self.run_at ||= self.class.db_time_now
196   - end
197   -
  193 + end
  194 +
198 195 end
199 196 end
4 lib/delayed/message_sending.rb
... ... @@ -1,7 +1,7 @@
1 1 module Delayed
2 2 module MessageSending
3   - def send_later(method, *args)
  3 + def send_later(method, *args)
4 4 Delayed::Job.enqueue Delayed::PerformableMethod.new(self, method.to_sym, args)
5 5 end
6   - end
  6 + end
7 7 end
18 lib/delayed/performable_method.rb
... ... @@ -1,22 +1,22 @@
1 1 module Delayed
2   - class PerformableMethod < Struct.new(:object, :method, :args)
  2 + class PerformableMethod < Struct.new(:object, :method, :args)
3 3 AR_STRING_FORMAT = /^AR\:([A-Z]\w+)\:(\d+)$/
4   -
  4 +
5 5 def initialize(object, method, args)
6 6 raise NoMethodError, "undefined method `#{method}' for #{self.inspect}" unless object.respond_to?(method)
7   -
  7 +
8 8 self.object = dump(object)
9 9 self.args = args.map { |a| dump(a) }
10 10 self.method = method.to_sym
11 11 end
12   -
  12 +
13 13 def perform
14 14 load(object).send(method, *args.map{|a| load(a)})
15 15 rescue ActiveRecord::RecordNotFound
16 16 # We cannot do anything about objects which were deleted in the meantime
17 17 true
18   - end
19   -
  18 + end
  19 +
20 20 private
21 21
22 22 def load(arg)
@@ -25,14 +25,14 @@ def load(arg)
25 25 else arg
26 26 end
27 27 end
28   -
  28 +
29 29 def dump(arg)
30 30 case arg
31 31 when ActiveRecord::Base then ar_to_string(arg)
32 32 else arg
33   - end
  33 + end
34 34 end
35   -
  35 +
36 36 def ar_to_string(obj)
37 37 "AR:#{obj.class}:#{obj.id}"
38 38 end
109 spec/delayed_method_spec.rb
... ... @@ -1,119 +1,110 @@
1 1 require File.dirname(__FILE__) + '/database'
2 2
3   -if not defined?(:ActiveRecord)
4   - module ActiveRecord
5   - class RecordNotFound < StandardError
6   - end
7   - end
8   -end
9   -
10   -
11 3 class SimpleJob
12   - cattr_accessor :runs; self.runs = 0
  4 + cattr_accessor :runs; self.runs = 0
13 5 def perform; @@runs += 1; end
14 6 end
15 7
16 8 class RandomRubyObject
17 9 def say_hello
18 10 'hello'
19   - end
20   -end
  11 + end
  12 +end
21 13
22 14 class ErrorObject
23   -
  15 +
24 16 def throw
25   - raise ActiveRecord::RecordNotFound, '...'
  17 + raise ActiveRecord::RecordNotFound, '...'
26 18 false
27 19 end
28   -
29   -end
  20 +
  21 +end
30 22
31 23 class StoryReader
32   -
  24 +
33 25 def read(story)
34   - "Epilog: #{story.tell}"
  26 + "Epilog: #{story.tell}"
35 27 end
36   -
  28 +
37 29 end
38 30
39 31 class StoryReader
40   -
  32 +
41 33 def read(story)
42   - "Epilog: #{story.tell}"
  34 + "Epilog: #{story.tell}"
43 35 end
44   -
  36 +
45 37 end
46 38
47   -
48 39 describe 'random ruby objects' do
49   -
  40 +
50 41 before { reset_db }
51 42
52 43 it "should respond_to :send_later method" do
53   -
54   - RandomRubyObject.new.respond_to?(:send_later)
55   -
56   - end
57   -
  44 +
  45 + RandomRubyObject.new.respond_to?(:send_later)
  46 +
  47 + end
  48 +
58 49 it "should raise a ArgumentError if send_later is called but the target method doesn't exist" do
59 50 lambda { RandomRubyObject.new.send_later(:method_that_deos_not_exist) }.should raise_error(NoMethodError)
60 51 end
61   -
62   - it "should add a new entry to the job table when send_later is called on it" do
  52 +
  53 + it "should add a new entry to the job table when send_later is called on it" do
63 54 Delayed::Job.count.should == 0
64   -
  55 +
65 56 RandomRubyObject.new.send_later(:to_s)
66 57
67 58 Delayed::Job.count.should == 1
68 59 end
69   -
  60 +
70 61 it "should run get the original method executed when the job is performed" do
71   -
  62 +
72 63 RandomRubyObject.new.send_later(:say_hello)
73   -
74   - Delayed::Job.count.should == 1
75   - end
  64 +
  65 + Delayed::Job.count.should == 1
  66 + end
76 67
77 68 it "should ignore ActiveRecord::RecordNotFound errors because they are permanent" do
78   -
79   - ErrorObject.new.send_later(:throw)
80 69
81   - Delayed::Job.count.should == 1
82   -
  70 + ErrorObject.new.send_later(:throw)
  71 +
  72 + Delayed::Job.count.should == 1
  73 +
83 74 output = nil
84   -
  75 +
85 76 Delayed::Job.reserve do |e|
86 77 output = e.perform
87 78 end
88   -
  79 +
89 80 output.should == true
90   -
91   - end
92   -
93   - it "should store the object as string if its an active record" do
94   - story = Story.create :text => 'Once upon...'
95   - story.send_later(:tell)
96   -
  81 +
  82 + end
  83 +
  84 + it "should store the object as string if its an active record" do
  85 + story = Story.create :text => 'Once upon...'
  86 + story.send_later(:tell)
  87 +
97 88 job = Delayed::Job.find(:first)
98 89 job.payload_object.class.should == Delayed::PerformableMethod
99 90 job.payload_object.object.should == 'AR:Story:1'
100 91 job.payload_object.method.should == :tell
101   - job.payload_object.args.should == []
  92 + job.payload_object.args.should == []
102 93 job.payload_object.perform.should == 'Once upon...'
103   - end
104   -
  94 + end
  95 +
105 96 it "should store arguments as string if they an active record" do
106   -
107   - story = Story.create :text => 'Once upon...'
108   -
109   - reader = StoryReader.new
  97 +
  98 + story = Story.create :text => 'Once upon...'
  99 +
  100 + reader = StoryReader.new
110 101 reader.send_later(:read, story)
111   -
  102 +
112 103 job = Delayed::Job.find(:first)
113 104 job.payload_object.class.should == Delayed::PerformableMethod
114 105 job.payload_object.method.should == :read
115 106 job.payload_object.args.should == ['AR:Story:1']
116   - job.payload_object.perform.should == 'Epilog: Once upon...'
  107 + job.payload_object.perform.should == 'Epilog: Once upon...'
117 108 end
118   -
  109 +
119 110 end
132 spec/job_spec.rb
... ... @@ -1,134 +1,120 @@
1 1 require File.dirname(__FILE__) + '/database'
2 2
3 3 class SimpleJob
4   - cattr_accessor :runs; self.runs = 0
  4 + cattr_accessor :runs; self.runs = 0
5 5 def perform; @@runs += 1; end
6   -end
  6 +end
7 7
8 8 class ErrorJob
9   - cattr_accessor :runs; self.runs = 0
10   - def perform; raise 'did not work'; end
  9 + cattr_accessor :runs; self.runs = 0
  10 + def perform; raise 'did not work'; end
11 11 end
12 12
13 13 describe Delayed::Job do
14   -
15   - before :each do
  14 +
  15 + before :each do
16 16 reset_db
17   - end
  17 + end
18 18
19 19 it "should set run_at automatically" do
20 20 Delayed::Job.create(:payload_object => ErrorJob.new ).run_at.should_not == nil
21   - end
  21 + end
22 22
23 23 it "should raise ArgumentError when handler doesn't respond_to :perform" do
24 24 lambda { Delayed::Job.enqueue(Object.new) }.should raise_error(ArgumentError)
25 25 end
26   -
  26 +
27 27 it "should increase count after enqueuing items" do
28   - Delayed::Job.enqueue SimpleJob.new
  28 + Delayed::Job.enqueue SimpleJob.new
29 29 Delayed::Job.count.should == 1
30 30 end
31   -
32   - it "should call perform on jobs when running work_off" do
  31 +
  32 + it "should call perform on jobs when running work_off" do
33 33 SimpleJob.runs.should == 0
34   -
35   - Delayed::Job.enqueue SimpleJob.new
  34 +
  35 + Delayed::Job.enqueue SimpleJob.new
36 36 Delayed::Job.work_off
37   -
38   - SimpleJob.runs.should == 1
39   - end
40   -
41   - it "should re-schedule by about 1 second at first and increment this more and more minutes when it fails to execute properly" do
42   - Delayed::Job.enqueue ErrorJob.new
43   - runner = Delayed::Job.work_off(1)
  37 +
  38 + SimpleJob.runs.should == 1
  39 + end
  40 +
  41 + it "should re-schedule by about 1 second at first and increment this more and more minutes when it fails to execute properly" do
  42 + Delayed::Job.enqueue ErrorJob.new
  43 + runner = Delayed::Job.work_off(1)
44 44
45 45 job = Delayed::Job.find(:first)
46 46 job.last_error.should == 'did not work'
47 47 job.attempts.should == 1
48   - job.run_at.should > Time.now
49   - job.run_at.should < Time.now + 6.minutes
50   - end
51   -
  48 + job.run_at.should > Time.now
  49 + job.run_at.should < Time.now + 6.minutes
  50 + end
  51 +
52 52 it "should raise an DeserializationError when the job class is totally unknown" do
53 53
54 54 job = Delayed::Job.new
55 55 job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
56 56
57   - lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
  57 + lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
58 58 end
59 59
60 60 it "should try to load the class when it is unknown at the time of the deserialization" do
61   - job = Delayed::Job.new
  61 + job = Delayed::Job.new
62 62 job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
63 63
64 64 job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
65   -
66   - lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
67   - end
68   -
  65 +
  66 + lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
  67 + end
  68 +
69 69 it "should try include the namespace when loading unknown objects" do
70   - job = Delayed::Job.new
  70 + job = Delayed::Job.new
71 71 job['handler'] = "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
72   - job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
73   - lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
74   - end
75   -
76   -
  72 + job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
  73 + lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
  74 + end
  75 +
77 76 it "should also try to load structs when they are unknown (raises TypeError)" do
78   - job = Delayed::Job.new
  77 + job = Delayed::Job.new
79 78 job['handler'] = "--- !ruby/struct:JobThatDoesNotExist {}"
80 79
81 80 job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
82   -
83   - lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
84   - end
85   -
  81 +
  82 + lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
  83 + end
  84 +
86 85 it "should try include the namespace when loading unknown structs" do
87   - job = Delayed::Job.new
  86 + job = Delayed::Job.new
88 87 job['handler'] = "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
89   - job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
90   - lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
91   - end
92   -
93   -
  88 + job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
  89 + lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
  90 + end
  91 +
94 92 describe "when another worker is already performing an task, it" do
95   -
  93 +
96 94 before :each do
97 95 Delayed::Job.worker_name = 'worker1'
98 96 @job = Delayed::Job.create :payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => Time.now.utc
99 97 end
100   -
101   - it "should not allow a second worker to get exclusive access" do
102   - lambda { @job.lock_exclusively! 4.hours, 'worker2' }.should raise_error(Delayed::Job::LockError)
103   - end
104   -
105   - it "should be able to get access to the task if it was started more then max_age ago" do
  98 +
  99 + it "should not allow a second worker to get exclusive access" do
  100 + lambda { @job.lock_exclusively! 4.hours, 'worker2' }.should raise_error(Delayed::Job::LockError)
  101 + end
  102 +
  103 + it "should be able to get access to the task if it was started more then max_age ago" do
106 104 @job.locked_at = 5.hours.ago
107 105 @job.save
108 106
109   - @job.lock_exclusively! 4.hours, 'worker2'
  107 + @job.lock_exclusively! 4.hours, 'worker2'
110 108 @job.reload
111 109 @job.locked_by.should == 'worker2'
112 110 @job.locked_at.should > 1.minute.ago
113 111 end
114 112
115   - it "should be able to get exclusive access again when the worker name is the same" do
116   - @job.lock_exclusively! Time.now + 20, 'worker1'
  113 + it "should be able to get exclusive access again when the worker name is the same" do
  114 + @job.lock_exclusively! Time.now + 20, 'worker1'
117 115 @job.lock_exclusively! Time.now + 21, 'worker1'
118   - @job.lock_exclusively! Time.now + 22, 'worker1'
119   - end
  116 + @job.lock_exclusively! Time.now + 22, 'worker1'
  117 + end
120 118 end
121   -
122   -end
123   -
124   -
125   -
126   -
127   -
128   -
129   -
130   -
131   -
132   -
133   -
134 119
  120 +end
20 spec/story_spec.rb
... ... @@ -1,18 +1,18 @@
1   -require File.dirname(__FILE__) + '/database'
  1 +require File.dirname(__FILE__) + '/database'
2 2
3 3 describe "A story" do
4   -
5   - before do
  4 +
  5 + before do
6 6 reset_db
7 7 Story.create :text => "Once upon a time..."
8 8 end
9   -
  9 +
10 10 it "should be shared" do
11   - Story.find(:first).tell.should == 'Once upon a time...'
12   - end
13   -
  11 + Story.find(:first).tell.should == 'Once upon a time...'
  12 + end
  13 +
14 14 it "should not return its result if it storytelling is delayed" do
15   - Story.find(:first).send_later(:tell).should_not == 'Once upon a time...'
16   - end
17   -
  15 + Story.find(:first).send_later(:tell).should_not == 'Once upon a time...'
  16 + end
  17 +
18 18 end

0 comments on commit 0581928

Please sign in to comment.
Something went wrong with that request. Please try again.