Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

* spawns background processing for downloading mailing and sending em…

…ail notifications

* zips mailboxes
* actionmailer support
  • Loading branch information...
commit 00711ca1cc1edc5cb1c914fc02dfb017e996db70 1 parent d6a9038
@hone authored
View
40 app/controllers/backups_controller.rb
@@ -14,22 +14,10 @@ def create
@pop3 = Pop3.new( params[:remote_mail] )
respond_to do |format|
if @pop3.valid?
- @pop3.setup_mailer
- result = @pop3.download
-
- case result[:status]
- when Pop3::OK_FLAG
- flash[:notice] = "Backup successfully created"
- Notify.deliver_success( @pop3.email_address, { :inbox => result[:mbox_name] } )
- format.html { redirect_to( :action => :show, :mbox => result[:mbox_name] ) }
- when Pop3::AUTHENTICATION_ERROR_FLAG
- flash[:notice] = "Could not create backup due to authentication problems"
- format.html { render :action => :new }
- # TODO add stuff here
- when Pop3::TIMEOUT_ERROR_FLAG
- flash[:notice] = "Could not create backup due to timeout error"
- format.html { render :action => :new }
+ spawn do
+ mail_download( @pop3 )
end
+ format.html { redirect_to :action => :show }
else
flash[:notice] = "Problems with backing up mail, invalid input"
format.html { render :action => :new }
@@ -38,15 +26,8 @@ def create
end
def show
- @download = params[:mbox]
-
respond_to do |format|
- if @download.nil? or @download.empty? or not check_files_exist( @download )
- flash[:notice] = 'No valid mbox specified'
- format.html { redirect_to( :action => :new ) }
- else
- format.html
- end
+ format.html
end
end
@@ -57,4 +38,17 @@ def check_files_exist( files )
end
end
+ def mail_download( pop3 )
+ pop3.setup_mailer
+ result = pop3.download
+ case result[:status]
+ when Pop3::OK_FLAG
+ Notify.deliver_success( @pop3.email_address, result[:mbox_name] )
+ when Pop3::AUTHENTICATION_ERROR_FLAG
+ Notify.deliver_authentication_problem( @pop3.email_address )
+ when Pop3::TIMEOUT_ERROR_FLAG
+ Notify.deliver_timeout_problem( @pop3.email_address )
+ end
+ end
+
end
View
39 app/models/pop3.rb
@@ -5,37 +5,7 @@ class Pop3 < RemoteMail
DEFAULT_PORT = 110
DEFAULT_SSL_PORT = 992
-# PORT_MIN = 1
-# PORT_MAX = 65535
-# MAX_CHAR_LENGTH = 50
-
-# OK_FLAG = :ok
-# AUTHENTICATION_ERROR_FLAG = :authentication_error
-# TIMEOUT_ERROR_FLAG = :timeout_error
-
-# FILE_DIR = File.join( RAILS_ROOT, 'public/download' )
-
- column :email_address, :string
- column :server , :string
- column :old_server, :string
- column :username , :string
- column :password , :string
- column :ssl , :boolean
- column :port , :integer
- column :old_port, :integer
-
- # validations
-# validates_presence_of :email_address
-# validates_presence_of :server
-# validates_presence_of :username
-# validates_presence_of :password
-# validates_numericality_of :port, :greater_than_or_equal_to => PORT_MIN, :less_than_or_equal_to => PORT_MAX
-# validates_format_of :email_address, :with => /^[\w.]+@\w+\.(\w+\.)*\w+$/
-# validates_format_of :server, :with => /^\w+\.\w+\.(\w+\.)*\w+$/
-# validates_length_of :email_address, :maximum => MAX_CHAR_LENGTH
-# validates_length_of :server, :maximum => MAX_CHAR_LENGTH
-# validates_length_of :username, :maximum => MAX_CHAR_LENGTH
-# validates_length_of :password, :maximum => MAX_CHAR_LENGTH
+ RemoteMailHelper::setup_columns( self )
attr_reader :mailer
@@ -76,6 +46,9 @@ def download
mail_count = @mailer.mails.size
mbox_name = generate_mbox_name
write_mbox( mbox_name )
+ mbox_zip = "#{mbox_name}.zip"
+ zip_output = "#{FILE_DIR}/#{mbox_name}.zip"
+ zip( "#{TMP_DIR}/#{mbox_name}", zip_output )
@mailer.finish
rescue Net::POPAuthenticationError
status = AUTHENTICATION_ERROR_FLAG
@@ -85,7 +58,7 @@ def download
{
:mail_count => mail_count,
- :mbox_name => mbox_name,
+ :mbox_name => mbox_zip,
:status => status
}
end
@@ -96,7 +69,7 @@ def generate_mbox_name
end
def write_mbox( name )
- filename = "#{FILE_DIR}/#{name}"
+ filename = "#{TMP_DIR}/#{name}"
if File.exist?( filename )
FileUtils.rm( filename )
end
View
41 app/models/remote_mail.rb
@@ -1,3 +1,21 @@
+require 'zip/zip'
+require 'zip/zipfilesystem'
+
+module RemoteMailHelper
+ def self.setup_columns( klass )
+ klass.class_eval do
+ column :email_address, :string
+ column :server , :string
+ column :old_server, :string
+ column :username , :string
+ column :password , :string
+ column :ssl , :boolean
+ column :port , :integer
+ column :old_port, :integer
+ end
+ end
+end
+
# interface for accessing RemoteMail
class RemoteMail < ActiveRecord::BaseWithoutTable
PORT_MIN = 1
@@ -8,16 +26,10 @@ class RemoteMail < ActiveRecord::BaseWithoutTable
AUTHENTICATION_ERROR_FLAG = :authentication_error
TIMEOUT_ERROR_FLAG = :timeout_error
- FILE_DIR = File.join( RAILS_ROOT, 'public/download' )
+ FILE_DIR = File.join( RAILS_ROOT, 'public', 'download' )
+ TMP_DIR = File.join( RAILS_ROOT, 'public', 'tmp' )
- column :email_address, :string
- column :server , :string
- column :old_server, :string
- column :username , :string
- column :password , :string
- column :ssl , :boolean
- column :port , :integer
- column :old_port, :integer
+ RemoteMailHelper::setup_columns( self )
# validations
validates_presence_of :email_address
@@ -35,4 +47,15 @@ class RemoteMail < ActiveRecord::BaseWithoutTable
def download
raise NotImplementedError.new
end
+
+ def zip( files, output_file )
+ Zip::ZipFile.open(output_file, Zip::ZipFile::CREATE) do |zipfile|
+ files.each do |file|
+ base_file = File.basename( file )
+ zipfile.add( base_file, file )
+ end
+ end
+
+ files.each {|file| FileUtils.rm( file ) }
+ end
end
View
5 app/views/backups/show.html.erb
@@ -1,3 +1,2 @@
-<h1>BackupMyMail Downloads</h1>
-<p>To download, right click on the link and left click on "Save as" or your browser's equivalent."</p>
-<p><%= link_to 'Download', "/download/#{@download}" %></p>
+<h1>Thank you</h1>
+<p>You should be receiving an e-mail shortly when your backup is finished.</p>
View
2  app/views/notify/success.text.plain.erb
@@ -1,2 +1,2 @@
Hello,
-Your backups are now available and can be accessed here: <%= "http://localhost:3000/download/#{@backup}" %>
+Your backups are now available and can be accessed here: http://localhost:3000/download/<%= @backup %>
View
2  config/environment.rb
@@ -83,7 +83,7 @@
:port => "587",
:domain => "gmail.com",
:authentication => :login,
- :user_name => "otherinbox",
+ :user_name => "otherinbox@hone.wornpath.net",
:password => "0th3r1nb0x"
}
end
View
53 spec/controllers/backups_controller_spec.rb
@@ -4,8 +4,6 @@ module BackupsControllerSpecHelper
def setup_create( result )
Pop3.should_receive(:new).once.and_return(@pop3)
@pop3.should_receive(:valid?).once.and_return(true)
- @pop3.should_receive(:setup_mailer).once
- @pop3.should_receive(:download).once.and_return( result )
end
end
@@ -32,45 +30,20 @@ def setup_create( result )
end
describe "GET 'show'" do
- before(:each) do
- @params = MBOX_NAME
- end
-
def do_get
- get :show, :mbox => @params
+ get :show
end
it "should be successful" do
- FileUtils.touch( MBOX_FILE )
do_get
- assigns[:download].should == MBOX_NAME
response.should be_success
response.should render_template( :show )
-
- remove_file( MBOX_FILE )
- end
-
- it "should redirect if no mbox is specified" do
- @params = nil
- do_get
-
- response.should redirect_to( :action => :new )
- flash[:notice].should match /no valid mbox/i
- end
-
- it "should redirect if mbox does not exist" do
- remove_file( MBOX_FILE )
- do_get
-
- assigns[:download].should == MBOX_NAME
- response.should redirect_to( :action => :new )
- flash[:notice].should match /no valid mbox/i
end
end
end
-describe BackupsController, " handling POST /backups" do
+describe BackupsController, " handling POST /backups/create" do
include BackupsControllerSpecHelper
before do
@@ -91,7 +64,7 @@ def do_post
post :create, :remote_mail => @params
end
- it "should redirect to page to download mbox" do
+ it "should redirect to page to show page" do
result =
{
:mail_count => 3,
@@ -99,11 +72,9 @@ def do_post
:status => Pop3::OK_FLAG
}
setup_create( result )
- @pop3.should_receive(:email_address).and_return(@params[:email_address])
do_post
- response.should redirect_to( :action => :show, :mbox => result[:mbox_name] )
- flash[:notice].should match /success/i
+ response.should redirect_to( :action => :show )
end
it "should render new page if invalid RemoteMail" do
@@ -116,7 +87,7 @@ def do_post
flash[:notice].should match /problems/i
end
- it "should show error for authentication problem" do
+ it "should redirect to show page for authentication problem" do
result =
{
:status => Pop3::AUTHENTICATION_ERROR_FLAG
@@ -124,21 +95,19 @@ def do_post
setup_create( result )
do_post
- response.should be_success
- response.should render_template(:new)
- flash[:notice].should match /authentication/i
+ response.should redirect_to( :action => 'show' )
end
- it "should show error for timeout problem" do
- result =
+ it "should redirect to show page for timeout problem" do
+ result =
{
:status => Pop3::TIMEOUT_ERROR_FLAG
}
setup_create( result )
do_post
- response.should be_success
- response.should render_template(:new)
- flash[:notice].should match /timeout/i
+ response.should redirect_to( :action => 'show' )
end
+
+ # TODO add examples for spec'ng spawn process
end
View
103 spec/models/pop3_spec.rb
@@ -2,7 +2,7 @@
module Pop3SpecHelper
def setup_pop3( opts = {}, setup_mailer = true )
- @valid_attributes = valid_pop3_attributes.merge( opts )
+ @valid_attributes = valid_attributes.merge( opts )
@pop3 = Pop3.new
@pop3.attributes = @valid_attributes
@@ -25,21 +25,6 @@ def setup_mock_time_pop3( opts = {}, setup_mailer = true )
@time.should_receive(:to_s).once.and_return( 'Thu Jan 08 01:22:01 -0500 2009' )
setup_pop3( opts, setup_mailer )
end
-
- def should_have_error_on_attribute( attribute, value = nil, error_num = 1 )
- setup_pop3( { attribute => value }, false )
-
- @pop3.should_not be_valid
- @pop3.should have(error_num).error_on(attribute)
- end
-
- def setup_long_variable( value, length )
- return_value = value
- difference = length - value.size + 1
- 1.upto( difference ) { return_value += 'a' }
-
- return_value
- end
end
describe Pop3, "setup mailer" do
@@ -128,18 +113,17 @@ def setup_long_variable( value, length )
old_mailer.should_not == @pop3.mailer
end
-
end
describe Pop3, "download mail" do
include Pop3SpecHelper
before(:each) do
- remove_file( MBOX_FILE )
+ remove_file( MBOX_FILE_ZIP )
end
after(:all) do
- remove_file( MBOX_FILE )
+ remove_file( MBOX_FILE_ZIP )
end
it "should not connect due to authentication problem" do
@@ -167,12 +151,14 @@ def setup_long_variable( value, length )
it "should download mail" do
pending( "this test should be run explicitly" )
setup_mock_time_pop3
+ puts "tmpdir: #{Dir::tmpdir}"
result = @pop3.download
result.should_not be_nil
result[:mail_count].should == 3
- result[:mbox_name].should == MBOX_NAME
+ result[:mbox_name].should == MBOX_ZIP
result[:status].should === Pop3::OK_FLAG
+ File.should be_exist( MBOX_FILE_ZIP )
end
end
@@ -180,11 +166,11 @@ def setup_long_variable( value, length )
include Pop3SpecHelper
before(:each) do
- remove_file( MBOX_FILE )
+ remove_file( TMP_MBOX_FILE )
end
after(:all) do
- remove_file( MBOX_FILE )
+ remove_file( TMP_MBOX_FILE )
end
it "should generate mbox name" do
@@ -198,11 +184,11 @@ def setup_long_variable( value, length )
@pop3.mailer.stub!(:mails).and_return([[1],[2],[3]])
@pop3.write_mbox( MBOX_NAME )
- File.should be_exist( MBOX_FILE )
+ File.should be_exist( TMP_MBOX_FILE )
end
it "should overwrite (truncate) an existing file of the same name" do
- File.open( MBOX_FILE, 'w' ) do |file|
+ File.open( TMP_MBOX_FILE, 'w' ) do |file|
file.puts "delete this line"
end
@@ -210,74 +196,9 @@ def setup_long_variable( value, length )
@pop3.mailer.stub!(:mails).and_return([[1],[2],[3]])
@pop3.write_mbox( MBOX_NAME )
- File.should be_exist( MBOX_FILE )
- File.open( MBOX_FILE ) do |file|
+ File.should be_exist( TMP_MBOX_FILE )
+ File.open( TMP_MBOX_FILE ) do |file|
/^delete this line/.match( file.readlines.first ).should be_nil
end
end
end
-
-describe Pop3, "validations" do
- include Pop3SpecHelper
-
- it "should create a valid Pop3" do
- setup_pop3( {}, false )
-
- @pop3.should be_valid
- end
-
- it "should require an email address" do
- should_have_error_on_attribute( :email_address, nil, 3 )
- end
-
- it "should require a server" do
- should_have_error_on_attribute( :server, nil, 3 )
- end
-
- it "should require a username" do
- should_have_error_on_attribute( :username, nil, 2 )
- end
-
- it "should require a password" do
- should_have_error_on_attribute( :password, nil, 2 )
- end
-
- it "should require port to be greater than or equal to port min" do
- should_have_error_on_attribute( :port, Pop3::PORT_MIN - 1 )
- end
-
- it "should require port to be less than or equal to port max" do
- should_have_error_on_attribute( :port, Pop3::PORT_MAX + 1 )
- end
-
- it "should require a somewhat valid email address" do
- should_have_error_on_attribute( :email_address, 'boo' )
- should_have_error_on_attribute( :email_address, 'boo@boo.boo.' )
- end
-
- it "should require a somewhat valid server" do
- should_have_error_on_attribute( :server, 'boo' )
- should_have_error_on_attribute( :server, 'boo.boo' )
- should_have_error_on_attribute( :server, 'boo.boo.boo.' )
- end
-
- it "should require email address to be less than or equal to max chars" do
- long_email = setup_long_variable( "boo@boo.com", Pop3::MAX_CHAR_LENGTH )
- should_have_error_on_attribute( :email_address, long_email )
- end
-
- it "should require server to be less than or equal to max chars" do
- long_server = setup_long_variable( "pop.gmail.com", Pop3::MAX_CHAR_LENGTH )
- should_have_error_on_attribute( :server, long_server )
- end
-
- it "should require username to be less than or equal to max chars" do
- long_username = setup_long_variable( "otherinbox", Pop3::MAX_CHAR_LENGTH )
- should_have_error_on_attribute( :username, long_username )
- end
-
- it "should require password to be less than or equal to max chars" do
- long_password = setup_long_variable( "password", Pop3::MAX_CHAR_LENGTH )
- should_have_error_on_attribute( :password, long_password )
- end
-end
View
108 spec/models/remote_mail_spec.rb
@@ -0,0 +1,108 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+module RemoteMailSpecHelper
+ def setup_remote_mail( options = {} )
+ @remote_mail = RemoteMail.new
+ @remote_mail.attributes = valid_attributes.merge( options )
+ end
+
+ def should_have_error_on_attribute( attribute, value = nil, error_num = 1 )
+ setup_remote_mail( { attribute => value } )
+
+ @remote_mail.should_not be_valid
+ @remote_mail.should have(error_num).error_on(attribute)
+ end
+end
+
+describe RemoteMail, "zip" do
+ include RemoteMailSpecHelper
+
+ before(:all) do
+ @zip_output = MBOX_FILE_ZIP
+ end
+
+ before(:each) do
+ setup_remote_mail
+ remove_file( TMP_MBOX_FILE )
+ end
+
+ it "should zip a single mbox" do
+ FileUtils.touch( TMP_MBOX_FILE )
+
+ @remote_mail.zip( TMP_MBOX_FILE, @zip_output )
+ File.should be_exist( @zip_output )
+ File.should_not be_exist( TMP_MBOX_FILE )
+ end
+
+ it "should overwrite existing zip file"
+
+ after(:all) do
+ remove_file( TMP_MBOX_FILE )
+ remove_file( @zip_output )
+ end
+end
+
+describe RemoteMail, "validations" do
+ include RemoteMailSpecHelper
+
+ it "should create a valid RemoteMail" do
+ setup_remote_mail
+
+ @remote_mail.should be_valid
+ end
+
+ it "should require an email address" do
+ should_have_error_on_attribute( :email_address, nil, 3 )
+ end
+
+ it "should require a server" do
+ should_have_error_on_attribute( :server, nil, 3 )
+ end
+
+ it "should require a username" do
+ should_have_error_on_attribute( :username, nil, 2 )
+ end
+
+ it "should require a password" do
+ should_have_error_on_attribute( :password, nil, 2 )
+ end
+
+ it "should require port to be greater than or equal to port min" do
+ should_have_error_on_attribute( :port, Pop3::PORT_MIN - 1 )
+ end
+
+ it "should require port to be less than or equal to port max" do
+ should_have_error_on_attribute( :port, Pop3::PORT_MAX + 1 )
+ end
+
+ it "should require a somewhat valid email address" do
+ should_have_error_on_attribute( :email_address, 'boo' )
+ should_have_error_on_attribute( :email_address, 'boo@boo.boo.' )
+ end
+
+ it "should require a somewhat valid server" do
+ should_have_error_on_attribute( :server, 'boo' )
+ should_have_error_on_attribute( :server, 'boo.boo' )
+ should_have_error_on_attribute( :server, 'boo.boo.boo.' )
+ end
+
+ it "should require email address to be less than or equal to max chars" do
+ long_email = setup_long_variable( "boo@boo.com", RemoteMail::MAX_CHAR_LENGTH )
+ should_have_error_on_attribute( :email_address, long_email )
+ end
+
+ it "should require server to be less than or equal to max chars" do
+ long_server = setup_long_variable( "pop.gmail.com", RemoteMail::MAX_CHAR_LENGTH )
+ should_have_error_on_attribute( :server, long_server )
+ end
+
+ it "should require username to be less than or equal to max chars" do
+ long_username = setup_long_variable( "otherinbox", RemoteMail::MAX_CHAR_LENGTH )
+ should_have_error_on_attribute( :username, long_username )
+ end
+
+ it "should require password to be less than or equal to max chars" do
+ long_password = setup_long_variable( "password", RemoteMail::MAX_CHAR_LENGTH )
+ should_have_error_on_attribute( :password, long_password )
+ end
+end
View
13 spec/spec_helper.rb
@@ -47,7 +47,10 @@
end
MBOX_NAME = 'bd4937b271d8f20c3003489a231b3824943a163f'
+MBOX_ZIP = "#{MBOX_NAME}.zip"
MBOX_FILE = File.join( RAILS_ROOT, 'public', 'download', MBOX_NAME )
+MBOX_FILE_ZIP = File.join( RAILS_ROOT, 'public', 'download', "#{MBOX_NAME}.zip" )
+TMP_MBOX_FILE = File.join( RAILS_ROOT, 'public', 'tmp', MBOX_NAME )
def remove_file( file )
if File.exist?( file )
@@ -61,7 +64,7 @@ def setup_mock_time
@time.stub!(:to_s).and_return( 'Thu Jan 08 01:22:01 -0500 2009' )
end
-def valid_pop3_attributes
+def valid_attributes
@valid_attributes = {
:email_address => 'test.otherinbox@gmail.com',
:server => 'pop.gmail.com',
@@ -71,3 +74,11 @@ def valid_pop3_attributes
:port => 995
}
end
+
+def setup_long_variable( value, length )
+ return_value = value
+ difference = length - value.size + 1
+ 1.upto( difference ) { return_value += 'a' }
+
+ return_value
+end
View
2  spec/views/backups/new.html.erb_spec.rb
@@ -3,7 +3,7 @@
describe "/backups/new" do
before(:each) do
@remote_mail = mock_model( RemoteMail )
- valid_pop3_attributes.each do |key, value|
+ valid_attributes.each do |key, value|
@remote_mail.stub!(key).and_return( value )
end
assigns[:remote_mail] = @remote_mail
View
4 spec/views/backups/show.html.erb_spec.rb
@@ -6,7 +6,7 @@
render 'backups/show'
end
- it "should show download link" do
- response.should have_tag( "a[href=?]", "/download/#{MBOX_NAME}", "Download" )
+ it "should show thank you" do
+ response.should have_text( /thank you/i )
end
end
View
51 vendor/plugins/spawn/CHANGELOG
@@ -0,0 +1,51 @@
+v0.1 - 2007/09/13
+
+initial version
+
+--------------------------------------------------
+v0.2 - 2007/09/28
+
+* return PID of the child process
+* added ":detach => false" option
+
+--------------------------------------------------
+v0.3 - 2007/10/15
+
+* added ':method => :thread' for threaded spawns
+* removed ':detach => false' option in favor of more generic implementation
+* added ability to set configuration of the form 'Spawn::method :thread'
+* added patch to ActiveRecord::Base to allow for more efficient reconnect in child processes
+* added monkey patch for http://dev.rubyonrails.org/ticket/7579
+* added wait() method to wait for spawned code blocks
+* don't allow threading if allow_concurrency=false
+
+--------------------------------------------------
+v0.4 - 2008/1/26
+
+* default to :thread on windows, still :fork on all other platforms
+* raise exception when used with :method=>:true and allow_concurrency != true
+
+--------------------------------------------------
+v0.5 - 2008/3/1
+* also default to :thread on JRuby (java)
+* added new :method => :yield which doesn't fork or thread, this is useful for testing
+* fixed problem with connections piling up on PostgreSQL
+
+--------------------------------------------------
+v0.6 - 2008/04/21
+* only apply clear_reloadable_connections patch on Rails 1.x (7579 fixed in Rails 2.x)
+* made it more responsive in more environments by disconnecting from the listener socket in the forked process
+
+--------------------------------------------------
+v0.7 - 2008/04/24
+* more generic mechanism for closing resources after fork
+* check for existence of Mongrel before patching it
+
+--------------------------------------------------
+v0.8 - 2008/05/02
+* call exit! within the ensure block so that at_exit handlers aren't called on exceptions
+* set logger from RAILS_DEFAULT_LOGGER if available, else STDERR
+
+--------------------------------------------------
+v0.9 - 2008/05/11
+* added ability to set nice level for child process
View
22 vendor/plugins/spawn/LICENSE
@@ -0,0 +1,22 @@
+Copyright (c) 2007 Tom Anderson (tom@squeat.com)
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
View
118 vendor/plugins/spawn/README
@@ -0,0 +1,118 @@
+Spawn
+=====
+
+This plugin provides a 'spawn' method to easily fork OR thread long-running sections of
+code so that your application can return results to your users more quickly.
+This plugin works by creating new database connections in ActiveRecord::Base for the
+spawned block.
+
+The plugin also patches ActiveRecord::Base to handle some known bugs when using
+threads (see lib/patches.rb).
+
+Usage
+-----
+
+Here's a simple example of how to demonstrate the spawn plugin.
+In one of your controllers, insert this code (after installing the plugin of course):
+
+ spawn do
+ logger.info("I feel sleepy...")
+ sleep 11
+ logger.info("Time to wake up!")
+ end
+
+If everything is working correctly, your controller should finish quickly then you'll see
+the last log message several seconds later.
+
+If you need to wait for the spawned processes/threads, then pass the objects returned by
+spawn to Spawn::wait(), like this:
+
+ N.times do |i|
+ # spawn N blocks of code
+ spawn_ids[i] = spawn do
+ something(i)
+ end
+ end
+ # wait for all N blocks of code to finish running
+ wait(spawn_ids)
+
+If you want your forked child to run at a lower priority than the parent process, pass in
+the :nice option like this:
+
+ spawn(:nice => 7) do
+ do_something_nicely
+ end
+
+By default, spawn will use the fork to spawn child processes. You can configure it to
+do threading either by telling the spawn method when you call it or by configuring your
+environment.
+For example, this is how you can tell spawn to use threading on the call,
+
+ spawn(:method => :thread) do
+ something
+ end
+
+When using the :thread setting, spawn will check to make sure that you have set
+allow_concurrency=true in your configuration. If you want this setting then
+put this line in one of your environment config files:
+
+ config.active_record.allow_concurrency = true
+
+If it is not set, then spawn will raise an exception.
+
+To (optionally) configure the spawn method in your configuration, add a line to
+your configuration file(s) like this:
+
+ Spawn::method :thread
+
+If you don't set any configuration, the :method will default to :fork. To
+specify different values for different environments, pass the environment as
+the 2nd argument:
+
+ Spawn::method :fork, 'production'
+ Spawn::method :yield, 'test'
+
+This allows you to set your production and development environments to use different
+methods according to your needs.
+
+Forking vs. Threading
+---------------------
+
+There are several tradeoffs for using threading vs. forking. Forking was chosen as the
+default primarily because it requires no configuration to get it working out of the box.
+
+Forking advantages:
+ - more reliable? - the ActiveRecord code is generally not deemed to be thread-safe.
+ Even though spawn attempts to patch known problems with the threaded implementation,
+ there are no guarantees. Forking is heavier but should be fairly reliable.
+ - keep running - this could also be a disadvantage, but you may find you want to fork
+ off a process that could have a life longer than its parent. For example, maybe you
+ want to restart your server without killing the spawned processes.
+ We don't necessarily condone this (i.e. haven't tried it) but it's technically possible.
+ - easier - forking works out of the box with spawn, threading requires you set
+ allow_concurrency=true. Also, beware of automatic reloading of classes in development
+ mode (config.cache_classes = false).
+
+Threading advantages:
+ - less filling - threads take less resources... how much less? it depends. Some
+ flavors of Unix are pretty efficient at forking so the threading advantage may not
+ be as big as you think... but then again, maybe it's more than you think. ;-)
+ - debugging - you can set breakpoints in your threads
+
+Acknowledgements
+----------------
+
+This plugin was initially inspired by Scott Persinger's blog post on how to use fork
+in rails for background processing.
+ http://geekblog.vodpod.com/?p=26
+
+Further inspiration for the threading implementation came from Jonathon Rochkind's
+blog post on threading in rails.
+ http://bibwild.wordpress.com/2007/08/28/threading-in-rails/
+
+Also thanks to all who have helped debug problems and suggest improvements including:
+ Ahmed Adam, Tristan Schneiter, Scott Haug, Andrew Garfield, Eugene Otto, Dan Sharp,
+ Garry Tan (2.2.x fixes)
+ <your name here>
+
+Copyright (c) 2007-08 Tom Anderson (tom@squeat.com), see LICENSE
View
5 vendor/plugins/spawn/init.rb
@@ -0,0 +1,5 @@
+require 'patches'
+
+ActiveRecord::Base.send :include, Spawn
+ActionController::Base.send :include, Spawn
+ActiveRecord::Observer.send :include, Spawn
View
63 vendor/plugins/spawn/lib/patches.rb
@@ -0,0 +1,63 @@
+# see activerecord/lib/active_record/connection_adaptors/abstract/connection_specification.rb
+class ActiveRecord::Base
+ # reconnect without disconnecting
+ if Rails::VERSION::MAJOR == 2 && Rails::VERSION::MINOR >= 2
+ def self.spawn_reconnect(klass=self)
+ @@connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+ establish_connection
+ end
+ else
+ def self.spawn_reconnect(klass=self)
+ spec = @@defined_connections[klass.name]
+ konn = active_connections[klass.name]
+ # remove from internal arrays before calling establish_connection so that
+ # the connection isn't disconnected when it calls AR::Base.remove_connection
+ @@defined_connections.delete_if { |key, value| value == spec }
+ active_connections.delete_if { |key, value| value == konn }
+ establish_connection(spec ? spec.config : nil)
+ end
+ end
+
+ # this patch not needed on Rails 2.x and later
+ if Rails::VERSION::MAJOR == 1
+ # monkey patch to fix threading problems,
+ # see: http://dev.rubyonrails.org/ticket/7579
+ def self.clear_reloadable_connections!
+ if @@allow_concurrency
+ # Hash keyed by thread_id in @@active_connections. Hash of hashes.
+ @@active_connections.each do |thread_id, conns|
+ conns.each do |name, conn|
+ if conn.requires_reloading?
+ conn.disconnect!
+ @@active_connections[thread_id].delete(name)
+ end
+ end
+ end
+ else
+ # Just one level hash, no concurrency.
+ @@active_connections.each do |name, conn|
+ if conn.requires_reloading?
+ conn.disconnect!
+ @@active_connections.delete(name)
+ end
+ end
+ end
+ end
+ end
+end
+
+# see mongrel/lib/mongrel.rb
+# it's possible that this is not defined if you're running outside of mongrel
+# examples: ./script/runner or ./script/console
+if defined? Mongrel::HttpServer
+ class Mongrel::HttpServer
+ # redefine Montrel::HttpServer::process_client so that we can intercept
+ # the socket that is being used so Spawn can close it upon forking
+ alias_method :orig_process_client, :process_client
+ def process_client(client)
+ Spawn.resource_to_close(client)
+ Spawn.resource_to_close(@socket)
+ orig_process_client(client)
+ end
+ end
+end
View
137 vendor/plugins/spawn/lib/spawn.rb
@@ -0,0 +1,137 @@
+module Spawn
+
+ # default to forking (unless windows or jruby)
+ @@method = (RUBY_PLATFORM =~ /(win32|java)/) ? :thread : :fork
+ # socket to close in child process
+ @@resources = []
+ # in some environments, logger isn't defined
+ @@logger = defined?(RAILS_DEFAULT_LOGGER) ? RAILS_DEFAULT_LOGGER : Logger.new(STDERR)
+
+ # add calls to this in your environment.rb to set your configuration, for example,
+ # to use forking everywhere except your 'development' environment:
+ # Spawn::method :fork
+ # Spawn::method :thread, 'development'
+ def self.method(method, env = nil)
+ if !env || env == RAILS_ENV
+ @@method = method
+ end
+ @@logger.debug "spawn> method = #{@@method}" if defined? RAILS_DEFAULT_LOGGER
+ end
+
+ # set the resource to disconnect from in the child process (when forking)
+ def self.resource_to_close(resource)
+ @@resources << resource
+ end
+
+ # close all the resources added by calls to resource_to_close
+ def self.close_resources
+ @@resources.each do |resource|
+ resource.close if resource && resource.respond_to?(:close) && !resource.closed?
+ end
+ @@resources = []
+ end
+
+ # Spawns a long-running section of code and returns the ID of the spawned process.
+ # By default the process will be a forked process. To use threading, pass
+ # :method => :thread or override the default behavior in the environment by setting
+ # 'Spawn::method :thread'.
+ def spawn(options = {})
+ options.symbolize_keys!
+ # setting options[:method] will override configured value in @@method
+ if options[:method] == :yield || @@method == :yield
+ yield
+ elsif options[:method] == :thread || (options[:method] == nil && @@method == :thread)
+ if ActiveRecord::Base.allow_concurrency
+ thread_it(options) { yield }
+ else
+ @@logger.error("spawn(:method=>:thread) only allowed when allow_concurrency=true")
+ raise "spawn requires config.active_record.allow_concurrency=true when used with :method=>:thread"
+ end
+ else
+ fork_it(options) { yield }
+ end
+ end
+
+ def wait(sids = [])
+ # wait for all threads and/or forks (if a single sid passed in, convert to array first)
+ Array(sids).each do |sid|
+ if sid.type == :thread
+ sid.handle.join()
+ else
+ begin
+ Process.wait(sid.handle)
+ rescue
+ # if the process is already done, ignore the error
+ end
+ end
+ end
+ # clean up connections from expired threads
+ ActiveRecord::Base.verify_active_connections!()
+ end
+
+ class SpawnId
+ attr_accessor :type
+ attr_accessor :handle
+ def initialize(t, h)
+ self.type = t
+ self.handle = h
+ end
+ end
+
+ protected
+ def fork_it(options)
+ # The problem with rails is that it only has one connection (per class),
+ # so when we fork a new process, we need to reconnect.
+ @@logger.debug "spawn> parent PID = #{Process.pid}"
+ child = fork do
+ begin
+ start = Time.now
+ @@logger.debug "spawn> child PID = #{Process.pid}"
+
+ # set the nice priority if needed
+ Process.setpriority(Process::PRIO_PROCESS, 0, options[:nice]) if options[:nice]
+
+ # disconnect from the listening socket, et al
+ Spawn.close_resources
+ # get a new connection so the parent can keep the original one
+ ActiveRecord::Base.spawn_reconnect
+
+ # run the block of code that takes so long
+ yield
+
+ rescue => ex
+ @@logger.error "spawn> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message}"
+ ensure
+ begin
+ # to be safe, catch errors on closing the connnections too
+ if Rails::VERSION::MAJOR >= 2 && Rails::VERSION::MINOR >= 2
+ ActiveRecord::Base.connection_handler.clear_all_connections!
+ else
+ ActiveRecord::Base.connection.disconnect!
+ ActiveRecord::Base.remove_connection
+ end
+ ensure
+ @@logger.info "spawn> child[#{Process.pid}] took #{Time.now - start} sec"
+ # this form of exit doesn't call at_exit handlers
+ exit!(0)
+ end
+ end
+ end
+
+ # detach from child process (parent may still wait for detached process if they wish)
+ Process.detach(child)
+
+ return SpawnId.new(:fork, child)
+ end
+
+ def thread_it(options)
+ # clean up stale connections from previous threads
+ ActiveRecord::Base.verify_active_connections!()
+ thr = Thread.new do
+ # run the long-running code block
+ yield
+ end
+ return SpawnId.new(:thread, thr)
+ end
+
+end
Please sign in to comment.
Something went wrong with that request. Please try again.