Skip to content

Commit

Permalink
Adds delayed job for mail
Browse files Browse the repository at this point in the history
  • Loading branch information
Shereef Bishay committed Jan 9, 2010
1 parent ff57ff8 commit 9f06e45
Show file tree
Hide file tree
Showing 35 changed files with 844 additions and 543 deletions.
2 changes: 1 addition & 1 deletion app/controllers/application_controller.rb
Expand Up @@ -137,7 +137,7 @@ def deny_access


# Authorize the user for the requested action # Authorize the user for the requested action
def authorize(ctrl = params[:controller], action = params[:action], global = false) def authorize(ctrl = params[:controller], action = params[:action], global = false)
logger.info("entering authorize") # logger.info("entering authorize")
allowed = User.current.allowed_to?({:controller => ctrl, :action => action}, @project, :global => global) allowed = User.current.allowed_to?({:controller => ctrl, :action => action}, @project, :global => global)
allowed ? true : deny_access allowed ? true : deny_access
end end
Expand Down
2 changes: 1 addition & 1 deletion app/models/document_observer.rb
Expand Up @@ -4,6 +4,6 @@


class DocumentObserver < ActiveRecord::Observer class DocumentObserver < ActiveRecord::Observer
def after_create(document) def after_create(document)
Mailer.deliver_document_added(document) if Setting.notified_events.include?('document_added') Mailer.send_later(:deliver_document_added,document) if Setting.notified_events.include?('document_added')
end end
end end
2 changes: 1 addition & 1 deletion app/models/issue_observer.rb
Expand Up @@ -4,6 +4,6 @@


class IssueObserver < ActiveRecord::Observer class IssueObserver < ActiveRecord::Observer
def after_create(issue) def after_create(issue)
Mailer.deliver_issue_add(issue) if Setting.notified_events.include?('issue_added') Mailer.send_later(:deliver_issue_add,issue) if Setting.notified_events.include?('issue_added')
end end
end end
2 changes: 1 addition & 1 deletion app/models/journal_observer.rb
Expand Up @@ -4,6 +4,6 @@


class JournalObserver < ActiveRecord::Observer class JournalObserver < ActiveRecord::Observer
def after_create(journal) def after_create(journal)
Mailer.deliver_issue_edit(journal) if Setting.notified_events.include?('issue_updated') Mailer.send_later(:deliver_issue_edit,journal) if Setting.notified_events.include?('issue_updated')
end end
end end
14 changes: 1 addition & 13 deletions app/models/mailer.rb
@@ -1,19 +1,6 @@
# BetterMeans - Work 2.0 # BetterMeans - Work 2.0
# Copyright (C) 2009 Shereef Bishay # Copyright (C) 2009 Shereef Bishay
# #
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.


class Mailer < ActionMailer::Base class Mailer < ActionMailer::Base
layout 'mailer' layout 'mailer'
Expand Down Expand Up @@ -285,6 +272,7 @@ def deliver!(mail = @mail)
end end
super(mail) super(mail)
end end



# Sends reminders to issue assignees # Sends reminders to issue assignees
# Available options: # Available options:
Expand Down
2 changes: 1 addition & 1 deletion app/models/message_observer.rb
Expand Up @@ -4,6 +4,6 @@


class MessageObserver < ActiveRecord::Observer class MessageObserver < ActiveRecord::Observer
def after_create(message) def after_create(message)
Mailer.deliver_message_posted(message) if Setting.notified_events.include?('message_posted') Mailer.send_later(:deliver_message_posted,message) if Setting.notified_events.include?('message_posted')
end end
end end
2 changes: 1 addition & 1 deletion app/models/news_observer.rb
Expand Up @@ -4,6 +4,6 @@


class NewsObserver < ActiveRecord::Observer class NewsObserver < ActiveRecord::Observer
def after_create(news) def after_create(news)
Mailer.deliver_news_added(news) if Setting.notified_events.include?('news_added') Mailer.send_later(:deliver_news_added,news) if Setting.notified_events.include?('news_added')
end end
end end
18 changes: 0 additions & 18 deletions db/migrate/20100109014849_create_delayed_jobs.rb

This file was deleted.

20 changes: 20 additions & 0 deletions db/migrate/20100109023733_create_delayed_jobs.rb
@@ -0,0 +1,20 @@
class CreateDelayedJobs < ActiveRecord::Migration
def self.up
create_table :delayed_jobs, :force => true do |table|
table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue
table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually.
table.text :handler # YAML-encoded string of the object that will do work
table.text :last_error # reason for last failure (See Note below)
table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
table.datetime :locked_at # Set when a client is working on this object
table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
table.string :locked_by # Who is working on this object (if locked)
table.timestamps
end

end

def self.down
drop_table :delayed_jobs
end
end
5 changes: 5 additions & 0 deletions script/delayed_job
@@ -0,0 +1,5 @@
#!/usr/bin/env ruby

require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
require 'delayed/command'
Delayed::Command.new(ARGV).daemonize
190 changes: 126 additions & 64 deletions vendor/plugins/delayed_job/README.textile
@@ -1,6 +1,6 @@
h1. Delayed::Job h1. Delayed::Job


Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. Delated_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background.


It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks. Amongst those tasks are: It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks. Amongst those tasks are:


Expand All @@ -12,101 +12,163 @@ It is a direct extraction from Shopify where the job table is responsible for a
* batch imports * batch imports
* spam checks * spam checks


h2. Setup h2. Installation


The library evolves around a delayed_jobs table which can be created by using: To install as a gem, add the following to @config/environment.rb@:
<pre><code>
script/generate delayed_job_migration
</code></pre>


The created table looks as follows: <pre>
config.gem 'collectiveidea-delayed_job', :lib => 'delayed_job',
:source => 'http://gems.github.com'
</pre>

Rake tasks are not automatically loaded from gems, so you'll need to add the following to your Rakefile:

<pre>
begin
require 'delayed/tasks'
rescue LoadError
STDERR.puts "Run `rake gems:install` to install delayed_job"
end
</pre>

To install as a plugin:

<pre>
script/plugin install git://github.com/collectiveidea/delayed_job.git
</pre>

After delayed_job is installed, run:

<pre>
script/generate delayed_job
rake db:migrate
</pre>

h2. Upgrading to 1.8

If you are upgrading from a previous release, you will need to generate the new @script/delayed_job@:

<pre>
script/generate delayed_job --skip-migration
</pre>

h2. Queuing Jobs

Call @#send_later(method, params)@ on any object and it will be processed in the background.

<pre>
# without delayed_job
Notifier.deliver_signup(@user)

# with delayed_job
Notifier.send_later :deliver_signup, @user
</pre>

If a method should always be run in the background, you can call @#handle_asynchronously@ after the method declaration:

<pre>
class Device
def deliver
# long running method
end
handle_asynchronously :deliver
end

device = Device.new
device.deliver
</pre>

h2. Running Jobs

@script/delayed_job@ can be used to manage a background process which will start working off jobs.

<pre>
$ RAILS_ENV=production script/delayed_job start
$ RAILS_ENV=production script/delayed_job stop

# Runs two workers in separate processes.
$ RAILS_ENV=production script/delayed_job -n 2 start
$ RAILS_ENV=production script/delayed_job stop
</pre>

Workers can be running on any computer, as long as they have access to the database and their clock is in sync. Keep in mind that each worker will check the database at least every 5 seconds.

You can also invoke @rake jobs:work@ which will start working off jobs. You can cancel the rake task with @CTRL-C@.

h2. Custom Jobs

Jobs are simple ruby objects with a method called perform. Any object which responds to perform can be stuffed into the jobs table. Job objects are serialized to yaml so that they can later be resurrected by the job runner.

<pre>
class NewsletterJob < Struct.new(:text, :emails)
def perform
emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) }
end
end

Delayed::Job.enqueue NewsletterJob.new('lorem ipsum...', Customers.find(:all).collect(&:email))
</pre>

h2. Gory Details

The library evolves around a delayed_jobs table which looks as follows:


<pre><code>
create_table :delayed_jobs, :force => true do |table| create_table :delayed_jobs, :force => true do |table|
table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue
table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually. table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually.
table.text :handler # YAML-encoded string of the object that will do work table.text :handler # YAML-encoded string of the object that will do work
table.string :last_error # reason for last failure (See Note below) table.text :last_error # reason for last failure (See Note below)
table.datetime :run_at # When to run. Could be Time.now for immediately, or sometime in the future. table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
table.datetime :locked_at # Set when a client is working on this object table.datetime :locked_at # Set when a client is working on this object
table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead) table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
table.string :locked_by # Who is working on this object (if locked) table.string :locked_by # Who is working on this object (if locked)
table.timestamps table.timestamps
end end
</code></pre>


On failure, the job is scheduled again in 5 seconds + N ** 4, where N is the number of retries. On failure, the job is scheduled again in 5 seconds + N ** 4, where N is the number of retries.


The default @MAX_ATTEMPTS@ is @25@. After this, the job either deleted (default), or left in the database with "failed_at" set. The default Worker.max_attempts is 25. After this, the job either deleted (default), or left in the database with "failed_at" set.
With the default of 25 attempts, the last retry will be 20 days later, with the last interval being almost 100 hours. With the default of 25 attempts, the last retry will be 20 days later, with the last interval being almost 100 hours.


The default @MAX_RUN_TIME@ is @4.hours@. If your job takes longer than that, another computer could pick it up. It's up to you to The default Worker.max_run_time is 4.hours. If your job takes longer than that, another computer could pick it up. It's up to you to
make sure your job doesn't exceed this time. You should set this to the longest time you think the job could take. make sure your job doesn't exceed this time. You should set this to the longest time you think the job could take.


By default, it will delete failed jobs (and it always deletes successful jobs). If you want to keep failed jobs, set By default, it will delete failed jobs (and it always deletes successful jobs). If you want to keep failed jobs, set
@Delayed::Job.destroy_failed_jobs = false@. The failed jobs will be marked with non-null failed_at. Delayed::Worker.destroy_failed_jobs = false. The failed jobs will be marked with non-null failed_at.


Here is an example of changing job parameters in Rails: Here is an example of changing job parameters in Rails:


<pre><code> <pre>
# config/initializers/delayed_job_config.rb # config/initializers/delayed_job_config.rb
Delayed::Job.destroy_failed_jobs = false Delayed::Worker.destroy_failed_jobs = false
silence_warnings do Delayed::Worker.sleep_delay = 60
Delayed::Job.const_set("MAX_ATTEMPTS", 3) Delayed::Worker.max_attempts = 3
Delayed::Job.const_set("MAX_RUN_TIME", 5.minutes) Delayed::Worker.max_run_time = 5.minutes
end </pre>
</code></pre>

Note: If your error messages are long, consider changing last_error field to a :text instead of a :string (255 character limit).



h2. Usage h3. Cleaning up

Jobs are simple ruby objects with a method called perform. Any object which responds to perform can be stuffed into the jobs table.
Job objects are serialized to yaml so that they can later be resurrected by the job runner.

<pre><code>
class NewsletterJob < Struct.new(:text, :emails)
def perform
emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) }
end
end

Delayed::Job.enqueue NewsletterJob.new('lorem ipsum...', Customers.find(:all).collect(&:email))
</code></pre>

There is also a second way to get jobs in the queue: send_later.


<pre><code> You can invoke @rake jobs:clear@ to delete all jobs in the queue.
BatchImporter.new(Shop.find(1)).send_later(:import_massive_csv, massive_csv)
</code></pre>


This will simply create a @Delayed::PerformableMethod@ job in the jobs table which serializes all the parameters you pass to it. There are some special smarts for active record objects h2. Mailing List
which are stored as their text representation and loaded from the database fresh when the job is actually run later.


h2. Running the jobs


You can invoke @rake jobs:work@ which will start working off jobs. You can cancel the rake task with @CTRL-C@. Join us on the mailing list at http://groups.google.com/group/delayed_job


You can also run by writing a simple @script/job_runner@, and invoking it externally: h2. How to contribute

<pre><code>
#!/usr/bin/env ruby
require File.dirname(__FILE__) + '/../config/environment'

Delayed::Worker.new.start
</code></pre>


Workers can be running on any computer, as long as they have access to the database and their clock is in sync. You can even If you find what looks like a bug:
run multiple workers on per computer, but you must give each one a unique name. (TODO: put in an example)
Keep in mind that each worker will check the database at least every 5 seconds.


Note: The rake task will exit if the database has any network connectivity problems. # Check the GitHub issue tracker to see if anyone else has had the same issue.
http://github.com/collectiveidea/delayed_job/issues/
# If you don't see anything, create an issue with information on how to reproduce it.


h3. Cleaning up If you want to contribute an enhancement or a fix:


You can invoke @rake jobs:clear@ to delete all jobs in the queue. # Fork the project on github.
http://github.com/collectiveidea/delayed_job/
# Make your changes with tests.
# Commit the changes without making changes to the Rakefile, VERSION, or any other files that aren't related to your enhancement or fix
# Send a pull request.


h3. Changes h3. Changes


Expand Down
38 changes: 38 additions & 0 deletions vendor/plugins/delayed_job/Rakefile
@@ -0,0 +1,38 @@
# -*- encoding: utf-8 -*-
begin
require 'jeweler'
rescue LoadError
puts "Jeweler not available. Install it with: sudo gem install jeweler"
exit 1
end

Jeweler::Tasks.new do |s|
s.name = "delayed_job"
s.summary = "Database-backed asynchronous priority queue system -- Extracted from Shopify"
s.email = "tobi@leetsoft.com"
s.homepage = "http://github.com/collectiveidea/delayed_job"
s.description = "Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks."
s.authors = ["Brandon Keepers", "Tobias Lütke"]

s.has_rdoc = true
s.rdoc_options = ["--main", "README.textile", "--inline-source", "--line-numbers"]
s.extra_rdoc_files = ["README.textile"]

s.test_files = Dir['spec/**/*']

s.add_development_dependency "rspec"
s.add_development_dependency "sqlite3-ruby"
end

require 'spec/rake/spectask'

task :default => :spec

desc 'Run the specs'
Spec::Rake::SpecTask.new(:spec) do |t|
t.libs << 'lib'
t.pattern = 'spec/**/*_spec.rb'
t.verbose = true
end
task :spec => :check_dependencies

0 comments on commit 9f06e45

Please sign in to comment.