0
@@ -0,0 +1,20 @@
0
+Copyright (c) 2005 Tobias Luetke
0
+
0
+Permission is hereby granted, free of charge, to any person obtaining
0
+a copy of this software and associated documentation files (the
0
+"Software"), to deal in the Software without restriction, including
0
+without limitation the rights to use, copy, modify, merge, publish,
0
+distribute, sublicense, and/or sell copies of the Software, and to
0
+permit persons to whom the Software is furnished to do so, subject to
0
+the following conditions:
0
+
0
+The above copyright notice and this permission notice shall be
0
+included in all copies or substantial portions of the Software.
0
+
0
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOa AND
0
+NONINFRINGEMENT. IN NO EVENT SaALL THE AUTHORS OR COPYRIGHT HOLDERS BE
0
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
0
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
0
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
0
\ No newline at end of file

0
@@ -0,0 +1,103 @@
0
+Delayed::Job
0
+============
0
+
0
+Delated_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. 
0
+
0
+It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks. Amongst those tasks are:
0
+
0
+* sending massive newsletters
0
+* image resizing
0
+* http downloads
0
+* updating smart collections
0
+* updating solr, our search server, after product changes
0
+* batch imports 
0
+* spam checks 
0
+            
0
+== Setup == 
0
+            
0
+The library evolves around a delayed_jobs table which looks as follows: 
0
+
0
+  create_table :delayed_jobs, :force => true do |table|
0
+    table.integer :priority, :default => 0
0
+    table.integer :attempts, :default => 0
0
+    table.text :handler
0
+    table.string :last_error
0
+    table.datetime :run_at   
0
+    table.timestamps    
0
+  end        
0
+  
0
+== Usage == 
0
+
0
+Jobs are simple ruby objects with a method called perform. Any object which responds to perform can be stuffed into the jobs table.
0
+Job objects are serialized to yaml so that they can later be resurrected by the job runner. 
0
+
0
+  class NewsletterJob < Struct.new(:text, :emails)
0
+    def perform
0
+      emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) }
0
+    end    
0
+  end  
0
+  
0
+  Delayed::Job.enqueue NewsletterJob.new('lorem ipsum...', Customers.find(:all).collect(&:email))
0
+           
0
+There is also a second way to get jobs in the queue: send_later. 
0
+
0
+  
0
+  BatchImporter.new(Shop.find(1)).send_later(:import_massive_csv, massive_csv)                                                    
0
+  
0
+
0
+This will simply create a Delayed::PerformableMethod job in the jobs table which serializes all the parameters you pass to it. There are some special smarts for active record objects
0
+which are stored as their text representation and loaded from the database fresh when the job is actually run later.
0
+                                                                                                                              
0
+                                                                                                                    
0
+== Running the tasks ==        
0
+
0
+You can invoke rake jobs:work which will start working off jobs. You can cancel the rake task by CTRL-C. 
0
+
0
+At Shopify we run the the tasks from a simple script/job_runner which is being invoked by runnit: 
0
+                                           
0
+  #!/usr/bin/env ruby
0
+  require File.dirname(__FILE__) + '/../config/environment'
0
+
0
+  SLEEP = 15
0
+  RESTART_AFTER = 1000
0
+
0
+  trap('TERM') { puts 'Exiting...'; $exit = true }
0
+  trap('INT')  { puts 'Exiting...'; $exit = true }
0
+                
0
+  # this script dies after several runs to prevent memory leaks. 
0
+  # runnit will immediately start it again. 
0
+  count, runs_left = 0, RESTART_AFTER
0
+
0
+  loop do
0
+  
0
+    count = 0
0
+                      
0
+    # this requires the locking plugin, also from jadedPixel
0
+    ActiveRecord::base.aquire_lock("jobs table worker", 10) do
0
+      puts 'got lock'
0
+    
0
+      realtime = Benchmark.realtime do 
0
+        count = Delayed::Job.work_off      
0
+      end
0
+    end
0
+  
0
+    runs_left -= 1
0
+  
0
+    break if $exit
0
+  
0
+    if count.zero? 
0
+      sleep(SLEEP)
0
+    else
0
+      status = "#{count} jobs completed at %.2f j/s ..." % [count / realtime]
0
+      RAILS_DEFAULT_LOGGER.info status
0
+      puts status
0
+    end
0
+    
0
+    if $exit or runs_left <= 0
0
+      break 
0
+    end
0
+  end     
0
+  
0
+== Todo == 
0
+                                                                                                                           
0
+Work out a locking mechanism which would allow several job runners to run at the same time, spreading the load between them.

0
@@ -0,0 +1,5 @@
0
+require File.dirname(__FILE__) + '/lib/delayed/message_sending'
0
+require File.dirname(__FILE__) + '/lib/delayed/performable_method'
0
+require File.dirname(__FILE__) + '/lib/delayed/job'
0
+
0
+Object.send(:include, Delayed::MessageSending)
0
\ No newline at end of file

0
@@ -0,0 +1,145 @@
0
+module Delayed
0
+
0
+  class DeserializationError < StandardError
0
+  end                                                                                                              
0
+
0
+  class Job < ActiveRecord::Base      
0
+    ParseObjectFromYaml = /\!ruby\/\w+\:([^\s]+)/
0
+    
0
+    set_table_name :delayed_jobs
0
+  
0
+    class Runner
0
+      attr_accessor :logger, :jobs
0
+      attr_accessor :runs, :success, :failure
0
+    
0
+      def initialize(jobs, logger = nil)
0
+        @jobs = jobs       
0
+        @logger = logger
0
+        self.runs = self.success = self.failure = 0
0
+      end
0
+      
0
+      def run
0
+        
0
+        ActiveRecord::Base.cache do 
0
+          ActiveRecord::Base.transaction do 
0
+            @jobs.each do |job|
0
+              self.runs += 1
0
+              begin
0
+                time = Benchmark.measure do
0
+                  job.perform              
0
+                  ActiveRecord::Base.uncached { job.destroy }
0
+                  self.success += 1
0
+                end
0
+                logger.debug "Executed job in #{time.real}"          
0
+              rescue DeserializationError, StandardError, RuntimeError => e
0
+                if logger
0
+                  logger.error "Job #{job.id}: #{e.class} #{e.message}" 
0
+                  logger.error e.backtrace.join("\n")                   
0
+                end
0
+                ActiveRecord::Base.uncached { job.reshedule e.message }            
0
+                self.failure += 1
0
+              end
0
+            end                               
0
+          end
0
+        end
0
+      
0
+        self
0
+      end
0
+    end
0
+   
0
+    def self.enqueue(object, priority = 0)
0
+      raise ArgumentError, 'Cannot enqueue items which do not respond to perform' unless object.respond_to?(:perform)
0
+    
0
+      Job.create(:handler => object, :priority => priority)    
0
+    end
0
+  
0
+    def handler=(object)
0
+      self['handler'] = object.to_yaml
0
+    end
0
+  
0
+    def handler           
0
+      @handler ||= deserialize(self['handler'])
0
+    end
0
+  
0
+    def perform
0
+      handler.perform
0
+    end
0
+  
0
+    def reshedule(message)    
0
+      self.attempts  += 1
0
+      self.run_at     = self.class.time_now + 5.minutes
0
+      self.last_error = message
0
+      save!
0
+    end
0
+  
0
+    def self.peek(limit = 1)
0
+      if limit == 1
0
+        find(:first, :order => "priority DESC, run_at ASC", :conditions => ['run_at <= ?', time_now])
0
+      else
0
+        find(:all, :order => "priority DESC, run_at ASC", :limit => limit, :conditions => ['run_at <= ?', time_now])
0
+      end
0
+    end
0
+    
0
+    def self.work_off(limit = 100)
0
+      jobs  = Job.find(:all, :conditions => ['run_at <= ?', time_now], :order => "priority DESC, run_at ASC", :limit => limit)    
0
+    
0
+      Job::Runner.new(jobs, logger).run
0
+    end
0
+  
0
+    protected  
0
+    
0
+    def self.time_now
0
+      (ActiveRecord::Base.default_timezone == :utc) ? Time.now.utc : Time.now           
0
+    end
0
+    
0
+    def before_save
0
+      self.run_at ||= self.class.time_now
0
+    end               
0
+    
0
+    private
0
+    
0
+    def deserialize(source)   
0
+      attempt_to_load_file = true
0
+      
0
+      begin 
0
+        handler = YAML.load(source) rescue nil                    
0
+        return handler if handler.respond_to?(:perform)  
0
+        
0
+        if handler.nil?
0
+          if source =~ ParseObjectFromYaml
0
+
0
+            # Constantize the object so that ActiveSupport can attempt 
0
+            # its auto loading magic. Will raise LoadError if not successful.
0
+            attempt_to_load($1)
0
+
0
+            # If successful, retry the yaml.load
0
+            handler = YAML.load(source)
0
+            return handler if handler.respond_to?(:perform)              
0
+          end
0
+        end
0
+        
0
+        if handler.is_a?(YAML::Object)
0
+          
0
+          # Constantize the object so that ActiveSupport can attempt 
0
+          # its auto loading magic. Will raise LoadError if not successful.
0
+          attempt_to_load(handler.class)
0
+        
0
+          # If successful, retry the yaml.load
0
+          handler = YAML.load(source)
0
+          return handler if handler.respond_to?(:perform)  
0
+        end
0
+                  
0
+        raise DeserializationError, 'Job failed to load: Unknown handler. Try to manually require the appropiate file.'   
0
+             
0
+      rescue TypeError, LoadError, NameError => e 
0
+        
0
+        raise DeserializationError, "Job failed to load: #{e.message}. Try to manually require the required file."         
0
+      end
0
+    end         
0
+    
0
+    def attempt_to_load(klass)
0
+       klass.constantize    
0
+    end
0
+  
0
+  end
0
+end
0
\ No newline at end of file

0
@@ -0,0 +1,7 @@
0
+module Delayed
0
+  module MessageSending
0
+    def send_later(method, *args)                                
0
+      Delayed::Job.enqueue Delayed::PerformableMethod.new(self, method.to_sym, args)
0
+    end
0
+  end  
0
+end
0
\ No newline at end of file

0
@@ -0,0 +1,37 @@
0
+module Delayed
0
+  class PerformableMethod < Struct.new(:object, :method, :args)    
0
+    AR_STRING_FORMAT = /^AR\:([A-Z]\w+)\:(\d+)$/
0
+    
0
+    def initialize(object, method, args)
0
+      raise NoMethodError, "undefined method `#{method}' for #{self.inspect}" unless object.respond_to?(method)
0
+      
0
+      self.object = dump(object)
0
+      self.args   = args.map { |a| dump(a) }
0
+      self.method = method.to_sym
0
+    end
0
+    
0
+    def perform
0
+      load(object).send(method, *args.map{|a| load(a)})
0
+    end               
0
+    
0
+    private
0
+
0
+    def load(arg)
0
+      case arg
0
+      when AR_STRING_FORMAT then $1.constantize.find($2)
0
+      else arg
0
+      end
0
+    end
0
+      
0
+    def dump(arg)
0
+      case arg
0
+      when ActiveRecord::Base then ar_to_string(arg)
0
+      else arg
0
+      end        
0
+    end
0
+        
0
+    def ar_to_string(obj)
0
+      "AR:#{obj.class}:#{obj.id}"
0
+    end    
0
+  end
0
+end
0
\ No newline at end of file

0
@@ -0,0 +1,33 @@
0
+$:.unshift(File.dirname(__FILE__) + '/../lib')
0
+                                     
0
+require 'rubygems'             
0
+require 'active_record'                     
0
+require File.dirname(__FILE__) + '/../init'
0
+
0
+ActiveRecord::Base.logger = Logger.new(nil)
0
+ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => '/tmp/jobs.sqlite')
0
+ActiveRecord::Migration.verbose = false
0
+          
0
+def reset_db
0
+  ActiveRecord::Schema.define do
0
+
0
+    create_table :delayed_jobs, :force => true do |table|
0
+      table.integer :priority, :default => 0
0
+      table.integer :attempts, :default => 0
0
+      table.text :handler
0
+      table.string :last_error
0
+      table.datetime :run_at   
0
+      table.timestamps    
0
+    end
0
+
0
+    create_table :stories, :force => true do |table|
0
+      table.string :text
0
+    end
0
+
0
+  end
0
+end                                         
0
+    
0
+# Purely useful for test cases...
0
+class Story < ActiveRecord::Base          
0
+  def tell; text; end
0
+end
0
\ No newline at end of file

0
@@ -0,0 +1,81 @@
0
+require File.dirname(__FILE__) + '/database'
0
+                     
0
+
0
+class SimpleJob
0
+  cattr_accessor :runs; self.runs = 0      
0
+  def perform; @@runs += 1; end
0
+end
0
+
0
+class RandomRubyObject  
0
+  def say_hello
0
+    'hello'
0
+  end  
0
+end        
0
+
0
+class StoryReader
0
+  
0
+  def read(story)
0
+    "Epilog: #{story.tell}"    
0
+  end
0
+  
0
+end
0
+
0
+
0
+describe 'random ruby objects' do
0
+  
0
+  before { reset_db }
0
+
0
+  it "should respond_to :send_later method" do
0
+                                           
0
+    RandomRubyObject.new.respond_to?(:send_later)   
0
+    
0
+  end     
0
+  
0
+  it "should raise a ArgumentError if send_later is called but the target method doesn't exist" do
0
+    lambda { RandomRubyObject.new.send_later(:method_that_deos_not_exist) }.should raise_error(NoMethodError)
0
+  end
0
+  
0
+  it "should add a new entry to the job table when send_later is called on it" do    
0
+    Delayed::Job.count.should == 0
0
+    
0
+    RandomRubyObject.new.send_later(:to_s)
0
+
0
+    Delayed::Job.count.should == 1
0
+  end
0
+  
0
+  it "should run get the original method executed when the job is performed" do
0
+    
0
+    RandomRubyObject.new.send_later(:say_hello)
0
+                               
0
+    Delayed::Job.count.should == 1        
0
+    Delayed::Job.peek.perform.should == 'hello'
0
+  end
0
+  
0
+  it "should store the object as string if its an active record" do
0
+    
0
+    story = Story.create :text => 'Once upon...'     
0
+    story.send_later(:tell)                      
0
+    
0
+    job =  Delayed::Job.peek 
0
+    job.handler.class.should   == Delayed::PerformableMethod
0
+    job.handler.object.should  == 'AR:Story:1'
0
+    job.handler.method.should  == :tell
0
+    job.handler.args.should    == []   
0
+    job.perform.should == 'Once upon...'
0
+  end 
0
+  
0
+  it "should store arguments as string if they an active record" do
0
+    
0
+    story = Story.create :text => 'Once upon...'     
0
+    
0
+    reader = StoryReader.new 
0
+    reader.send_later(:read, story)
0
+    
0
+    job =  Delayed::Job.peek 
0
+    job.handler.class.should   == Delayed::PerformableMethod
0
+    job.handler.method.should  == :read
0
+    job.handler.args.should    == ['AR:Story:1']
0
+    job.perform.should == 'Epilog: Once upon...'     
0
+  end
0
+  
0
+end
0
\ No newline at end of file

0
@@ -0,0 +1,122 @@
0
+require File.dirname(__FILE__) + '/database'
0
+
0
+
0
+class SimpleJob
0
+  cattr_accessor :runs; self.runs = 0      
0
+  def perform; @@runs += 1; end
0
+end                                     
0
+
0
+class ErrorJob
0
+  cattr_accessor :runs; self.runs = 0      
0
+  def perform; raise 'did not work'; end  
0
+end
0
+
0
+describe Delayed::Job do
0
+  
0
+  before :each do 
0
+    reset_db
0
+  end                  
0
+
0
+  it "should set run_at automatically" do
0
+    Delayed::Job.create.run_at.should_not == nil
0
+  end 
0
+
0
+  it "should raise ArgumentError when handler doesn't respond_to :perform" do
0
+    lambda { Delayed::Job.enqueue(Object.new) }.should raise_error(ArgumentError)
0
+  end
0
+      
0
+  it "should increase count after enqueuing items" do
0
+    Delayed::Job.enqueue SimpleJob.new      
0
+    Delayed::Job.count.should == 1
0
+  end
0
+  
0
+  it "should return nil when peeking on empty table" do      
0
+    Delayed::Job.peek.should == nil
0
+  end
0
+  
0
+  it "should return a job when peeking a table with jobs in it" do
0
+    Delayed::Job.enqueue SimpleJob.new      
0
+    Delayed::Job.peek.class.should == Delayed::Job
0
+  end              
0
+
0
+  it "should return an array of jobs when peek is called with a count larger than zero" do
0
+    Delayed::Job.enqueue SimpleJob.new      
0
+    Delayed::Job.peek(2).class.should == Array
0
+  end              
0
+  
0
+  it "should call perform on jobs when running work_off" do        
0
+    SimpleJob.runs.should == 0
0
+        
0
+    Delayed::Job.enqueue SimpleJob.new    
0
+    Delayed::Job.work_off(1)    
0
+    
0
+    SimpleJob.runs.should == 1   
0
+  end                         
0
+  
0
+  it "should re-schedule by about 5 minutes when it fails to execute properly" do            
0
+    Delayed::Job.enqueue ErrorJob.new    
0
+    runner = Delayed::Job.work_off(1)        
0
+    runner.success.should == 0
0
+    runner.failure.should == 1               
0
+    
0
+    job = Delayed::Job.find(:first)
0
+    job.last_error.should == 'did not work'
0
+    job.attempts.should == 1
0
+    job.run_at.should > Time.now + 4.minutes
0
+    job.run_at.should < Time.now + 6.minutes    
0
+  end                    
0
+  
0
+  it "should raise an DeserializationError when the job class is totally unknown" do
0
+
0
+    job = Delayed::Job.new 
0
+    job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
0
+
0
+    lambda { job.perform }.should raise_error(Delayed::DeserializationError)    
0
+  end
0
+
0
+  it "should try to load the class when it is unknown at the time of the deserialization" do
0
+    job = Delayed::Job.new 
0
+    job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
0
+
0
+    job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
0
+     
0
+    lambda { job.perform }.should raise_error(Delayed::DeserializationError)    
0
+  end                  
0
+  
0
+  it "should try include the namespace when loading unknown objects" do
0
+    job = Delayed::Job.new 
0
+    job['handler'] = "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
0
+    job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)     
0
+    lambda { job.perform }.should raise_error(Delayed::DeserializationError)    
0
+  end                  
0
+  
0
+  
0
+  it "should also try to load structs when they are unknown (raises TypeError)" do
0
+    job = Delayed::Job.new 
0
+    job['handler'] = "--- !ruby/struct:JobThatDoesNotExist {}"
0
+
0
+    job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
0
+     
0
+    lambda { job.perform }.should raise_error(Delayed::DeserializationError)    
0
+  end          
0
+  
0
+  it "should try include the namespace when loading unknown structs" do
0
+    job = Delayed::Job.new 
0
+    job['handler'] = "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
0
+    job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)     
0
+    lambda { job.perform }.should raise_error(Delayed::DeserializationError)    
0
+  end                  
0
+  
0
+end
0
+
0
+
0
+
0
+
0
+
0
+
0
+
0
+
0
+
0
+
0
+
0
+

0
@@ -0,0 +1,18 @@
0
+require File.dirname(__FILE__) + '/database'       
0
+
0
+describe "A story" do
0
+  
0
+  before do 
0
+    reset_db
0
+    Story.create :text => "Once upon a time..."
0
+  end
0
+  
0
+  it "should be shared" do
0
+    Story.find(:first).tell.should == 'Once upon a time...'    
0
+  end                                                      
0
+  
0
+  it "should not return its result if it storytelling is delayed" do
0
+    Story.find(:first).send_later(:tell).should_not == 'Once upon a time...'    
0
+  end                            
0
+    
0
+end
0
\ No newline at end of file

0
@@ -0,0 +1,29 @@
0
+namespace :jobs do
0
+               
0
+  task :work => :environment do 
0
+    
0
+    SLEEP = 5
0
+     
0
+    trap('TERM') { puts 'Exiting...'; $exit = true }
0
+    trap('INT')  { puts 'Exiting...'; $exit = true }
0
+
0
+    loop do
0
+      
0
+      count = 0
0
+
0
+      realtime = Benchmark.realtime do 
0
+        count = Delayed::Job.work_off
0
+      end
0
+
0
+      break if $exit
0
+
0
+      if count.zero? 
0
+        sleep(SLEEP)
0
+      else
0
+        RAILS_DEFAULT_LOGGER.info "#{count} jobs completed at %.2f j/s ..." % [count / realtime]
0
+      end
0
+      
0
+      break if $exit 
0
+    end
0
+  end
0
+end
0
\ No newline at end of file