Skip to content
Browse files

install workling and spawn plugins

  • Loading branch information...
1 parent c317d1a commit f074ee7ce815bc208293cc2812a5847982465987 @crossblaim crossblaim committed Jul 17, 2009
Showing with 2,831 additions and 0 deletions.
  1. +16 −0 config/workling.yml
  2. +11 −0 script/bj_invoker.rb
  3. +37 −0 script/starling_status.rb
  4. +17 −0 script/workling_client
  5. +7 −0 script/workling_starling_client
  6. +51 −0 vendor/plugins/spawn/CHANGELOG
  7. +22 −0 vendor/plugins/spawn/LICENSE
  8. +120 −0 vendor/plugins/spawn/README
  9. +5 −0 vendor/plugins/spawn/init.rb
  10. +105 −0 vendor/plugins/spawn/lib/patches.rb
  11. +141 −0 vendor/plugins/spawn/lib/spawn.rb
  12. +76 −0 vendor/plugins/workling/CHANGES.markdown
  13. +20 −0 vendor/plugins/workling/MIT-LICENSE
  14. +382 −0 vendor/plugins/workling/README.markdown
  15. +22 −0 vendor/plugins/workling/Rakefile
  16. +26 −0 vendor/plugins/workling/TODO.markdown
  17. +16 −0 vendor/plugins/workling/config/workling.yml
  18. +3 −0 vendor/plugins/workling/init.rb
  19. +17 −0 vendor/plugins/workling/install.rb
  20. +11 −0 vendor/plugins/workling/lib/rude_q/client.rb
  21. +150 −0 vendor/plugins/workling/lib/workling.rb
  22. +59 −0 vendor/plugins/workling/lib/workling/base.rb
  23. +40 −0 vendor/plugins/workling/lib/workling/clients/amqp_client.rb
  24. +54 −0 vendor/plugins/workling/lib/workling/clients/base.rb
  25. +82 −0 vendor/plugins/workling/lib/workling/clients/memcache_queue_client.rb
  26. +14 −0 vendor/plugins/workling/lib/workling/discovery.rb
  27. +42 −0 vendor/plugins/workling/lib/workling/remote.rb
  28. +124 −0 vendor/plugins/workling/lib/workling/remote/invokers/base.rb
  29. +41 −0 vendor/plugins/workling/lib/workling/remote/invokers/basic_poller.rb
  30. +37 −0 vendor/plugins/workling/lib/workling/remote/invokers/eventmachine_subscriber.rb
  31. +150 −0 vendor/plugins/workling/lib/workling/remote/invokers/threaded_poller.rb
  32. +35 −0 vendor/plugins/workling/lib/workling/remote/runners/backgroundjob_runner.rb
  33. +42 −0 vendor/plugins/workling/lib/workling/remote/runners/base.rb
  34. +45 −0 vendor/plugins/workling/lib/workling/remote/runners/client_runner.rb
  35. +23 −0 vendor/plugins/workling/lib/workling/remote/runners/not_remote_runner.rb
  36. +38 −0 vendor/plugins/workling/lib/workling/remote/runners/spawn_runner.rb
  37. +13 −0 vendor/plugins/workling/lib/workling/remote/runners/starling_runner.rb
  38. +37 −0 vendor/plugins/workling/lib/workling/return/store/base.rb
  39. +26 −0 vendor/plugins/workling/lib/workling/return/store/memory_return_store.rb
  40. +31 −0 vendor/plugins/workling/lib/workling/return/store/starling_return_store.rb
  41. +13 −0 vendor/plugins/workling/lib/workling/routing/base.rb
  42. +55 −0 vendor/plugins/workling/lib/workling/routing/class_and_method_routing.rb
  43. +11 −0 vendor/plugins/workling/script/bj_invoker.rb
  44. +48 −0 vendor/plugins/workling/script/listen.rb
  45. +37 −0 vendor/plugins/workling/script/starling_status.rb
  46. +17 −0 vendor/plugins/workling/script/workling_client
  47. +7 −0 vendor/plugins/workling/script/workling_starling_client
  48. +18 −0 vendor/plugins/workling/test/class_and_method_routing_test.rb
  49. +36 −0 vendor/plugins/workling/test/clients/memory_queue_client.rb
  50. +13 −0 vendor/plugins/workling/test/discovery_test.rb
  51. +29 −0 vendor/plugins/workling/test/invoker_basic_poller_test.rb
  52. +26 −0 vendor/plugins/workling/test/invoker_eventmachine_subscription_test.rb
  53. +34 −0 vendor/plugins/workling/test/invoker_threaded_poller_test.rb
  54. +36 −0 vendor/plugins/workling/test/memcachequeue_client_test.rb
  55. +23 −0 vendor/plugins/workling/test/memory_return_store_test.rb
  56. +9 −0 vendor/plugins/workling/test/mocks/client.rb
  57. +5 −0 vendor/plugins/workling/test/mocks/logger.rb
  58. +5 −0 vendor/plugins/workling/test/mocks/spawn.rb
  59. +11 −0 vendor/plugins/workling/test/not_remote_runner_test.rb
  60. +50 −0 vendor/plugins/workling/test/remote_runner_test.rb
  61. +18 −0 vendor/plugins/workling/test/return_store_test.rb
  62. +22 −0 vendor/plugins/workling/test/runners/thread_runner.rb
  63. +10 −0 vendor/plugins/workling/test/spawn_runner_test.rb
  64. +29 −0 vendor/plugins/workling/test/starling_return_store_test.rb
  65. +8 −0 vendor/plugins/workling/test/starling_runner_test.rb
  66. +48 −0 vendor/plugins/workling/test/test_helper.rb
  67. +10 −0 vendor/plugins/workling/test/workers/analytics/invites.rb
  68. +15 −0 vendor/plugins/workling/test/workers/util.rb
View
16 config/workling.yml
@@ -0,0 +1,16 @@
+# By default, NotRemoteRunner is used when RAILS_ENV == 'test'.
+#
+# You can pass options to memcached client by nesting the key value pairs
+# under 'memcache_options'.
+#
+# You can also use a cluster of Starlings. Simply give a comma separated
+# list of server:port, server:port, server:port values to listens_on.
+#
+production:
+ listens_on: localhost:15151
+
+development:
+ listens_on: localhost:22122
+
+test:
+ listens_on: localhost:12345
View
11 script/bj_invoker.rb
@@ -0,0 +1,11 @@
+@routing = Workling::Routing::ClassAndMethodRouting.new
+unnormalized = REXML::Text::unnormalize(STDIN.read)
+message, command, args = *unnormalized.match(/(^[^ ]*) (.*)/)
+options = Hash.from_xml(args)["hash"]
+
+if workling = @routing[command]
+ options = options.symbolize_keys
+ method_name = @routing.method_name(command)
+
+ workling.dispatch_to_worker_method(method_name, options)
+end
View
37 script/starling_status.rb
@@ -0,0 +1,37 @@
+require 'pp'
+
+puts '=> Loading Rails...'
+
+require File.dirname(__FILE__) + '/../config/environment'
+require File.dirname(__FILE__) + '/../vendor/plugins/workling/lib/workling/remote/invokers/basic_poller'
+require File.dirname(__FILE__) + '/../vendor/plugins/workling/lib/workling/routing/class_and_method_routing'
+
+puts '** Rails loaded.'
+
+trap(:INT) { exit }
+
+client = Workling::Clients::MemcacheQueueClient.new
+
+begin
+ client.connect
+ client.reset
+
+ client.stats # do this so that connection is shown as established below.
+
+ puts "Queue state:"
+ pp client.inspect
+ pp "Active?: #{client.active?}"
+ pp "Read Only?: #{client.readonly?}"
+ puts ""
+ puts "Servers:"
+ pp client.servers
+ puts ""
+ puts "Queue stats:"
+ pp client.stats
+
+ puts "\nThread Stats:"
+ pp Thread.list
+ensure
+ puts '** Exiting'
+ client.close
+end
View
17 script/workling_client
@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'daemons'
+
+workling = File.join(File.dirname(__FILE__), '..', 'vendor', 'plugins', 'workling', 'script', 'listen.rb')
+options = {
+ :app_name => "workling",
+ :ARGV => ARGV,
+ :dir_mode => :normal,
+ :dir => File.join(File.dirname(__FILE__), '..', 'log'),
+ :log_output => true,
+ :multiple => false,
+ :backtrace => true,
+ :monitor => true
+}
+
+Daemons.run(workling, options)
View
7 script/workling_starling_client
@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+puts "\n>>>>>"
+puts ">>>>> This script has been DEPRACATED. Please use script/workling_client instead!"
+puts ">>>>>\n\n"
+
+load File.dirname(__FILE__) + '/workling_client'
View
51 vendor/plugins/spawn/CHANGELOG
@@ -0,0 +1,51 @@
+v0.1 - 2007/09/13
+
+initial version
+
+--------------------------------------------------
+v0.2 - 2007/09/28
+
+* return PID of the child process
+* added ":detach => false" option
+
+--------------------------------------------------
+v0.3 - 2007/10/15
+
+* added ':method => :thread' for threaded spawns
+* removed ':detach => false' option in favor of more generic implementation
+* added ability to set configuration of the form 'Spawn::method :thread'
+* added patch to ActiveRecord::Base to allow for more efficient reconnect in child processes
+* added monkey patch for http://dev.rubyonrails.org/ticket/7579
+* added wait() method to wait for spawned code blocks
+* don't allow threading if allow_concurrency=false
+
+--------------------------------------------------
+v0.4 - 2008/1/26
+
+* default to :thread on windows, still :fork on all other platforms
+* raise exception when used with :method=>:true and allow_concurrency != true
+
+--------------------------------------------------
+v0.5 - 2008/3/1
+* also default to :thread on JRuby (java)
+* added new :method => :yield which doesn't fork or thread, this is useful for testing
+* fixed problem with connections piling up on PostgreSQL
+
+--------------------------------------------------
+v0.6 - 2008/04/21
+* only apply clear_reloadable_connections patch on Rails 1.x (7579 fixed in Rails 2.x)
+* made it more responsive in more environments by disconnecting from the listener socket in the forked process
+
+--------------------------------------------------
+v0.7 - 2008/04/24
+* more generic mechanism for closing resources after fork
+* check for existence of Mongrel before patching it
+
+--------------------------------------------------
+v0.8 - 2008/05/02
+* call exit! within the ensure block so that at_exit handlers aren't called on exceptions
+* set logger from RAILS_DEFAULT_LOGGER if available, else STDERR
+
+--------------------------------------------------
+v0.9 - 2008/05/11
+* added ability to set nice level for child process
View
22 vendor/plugins/spawn/LICENSE
@@ -0,0 +1,22 @@
+Copyright (c) 2007 Tom Anderson (tom@squeat.com)
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
View
120 vendor/plugins/spawn/README
@@ -0,0 +1,120 @@
+Spawn
+=====
+
+This plugin provides a 'spawn' method to easily fork OR thread long-running sections of
+code so that your application can return results to your users more quickly.
+This plugin works by creating new database connections in ActiveRecord::Base for the
+spawned block.
+
+The plugin also patches ActiveRecord::Base to handle some known bugs when using
+threads (see lib/patches.rb).
+
+Usage
+-----
+
+Here's a simple example of how to demonstrate the spawn plugin.
+In one of your controllers, insert this code (after installing the plugin of course):
+
+ spawn do
+ logger.info("I feel sleepy...")
+ sleep 11
+ logger.info("Time to wake up!")
+ end
+
+If everything is working correctly, your controller should finish quickly then you'll see
+the last log message several seconds later.
+
+If you need to wait for the spawned processes/threads, then pass the objects returned by
+spawn to Spawn::wait(), like this:
+
+ N.times do |i|
+ # spawn N blocks of code
+ spawn_ids[i] = spawn do
+ something(i)
+ end
+ end
+ # wait for all N blocks of code to finish running
+ wait(spawn_ids)
+
+If you want your forked child to run at a lower priority than the parent process, pass in
+the :nice option like this:
+
+ spawn(:nice => 7) do
+ do_something_nicely
+ end
+
+By default, spawn will use the fork to spawn child processes. You can configure it to
+do threading either by telling the spawn method when you call it or by configuring your
+environment.
+For example, this is how you can tell spawn to use threading on the call,
+
+ spawn(:method => :thread) do
+ something
+ end
+
+When using the :thread setting, spawn will check to make sure that you have set
+allow_concurrency=true in your configuration. If you want this setting then
+put this line in one of your environment config files:
+
+ config.active_record.allow_concurrency = true
+
+If it is not set, then spawn will raise an exception.
+
+To (optionally) configure the spawn method in your configuration, add a line to
+your configuration file(s) like this:
+
+ Spawn::method :thread
+
+If you don't set any configuration, the :method will default to :fork. To
+specify different values for different environments, pass the environment as
+the 2nd argument:
+
+ Spawn::method :fork, 'production'
+ Spawn::method :yield, 'test'
+
+This allows you to set your production and development environments to use different
+methods according to your needs.
+
+Forking vs. Threading
+---------------------
+
+There are several tradeoffs for using threading vs. forking. Forking was chosen as the
+default primarily because it requires no configuration to get it working out of the box.
+
+Forking advantages:
+ - more reliable? - the ActiveRecord code is generally not deemed to be thread-safe.
+ Even though spawn attempts to patch known problems with the threaded implementation,
+ there are no guarantees. Forking is heavier but should be fairly reliable.
+ - keep running - this could also be a disadvantage, but you may find you want to fork
+ off a process that could have a life longer than its parent. For example, maybe you
+ want to restart your server without killing the spawned processes.
+ We don't necessarily condone this (i.e. haven't tried it) but it's technically possible.
+ - easier - forking works out of the box with spawn, threading requires you set
+ allow_concurrency=true. Also, beware of automatic reloading of classes in development
+ mode (config.cache_classes = false).
+
+Threading advantages:
+ - less filling - threads take less resources... how much less? it depends. Some
+ flavors of Unix are pretty efficient at forking so the threading advantage may not
+ be as big as you think... but then again, maybe it's more than you think. ;-)
+ - debugging - you can set breakpoints in your threads
+
+Acknowledgements
+----------------
+
+This plugin was initially inspired by Scott Persinger's blog post on how to use fork
+in rails for background processing.
+ http://geekblog.vodpod.com/?p=26
+
+Further inspiration for the threading implementation came from Jonathon Rochkind's
+blog post on threading in rails.
+ http://bibwild.wordpress.com/2007/08/28/threading-in-rails/
+
+Also thanks to all who have helped debug problems and suggest improvements including:
+ Ahmed Adam, Tristan Schneiter, Scott Haug, Andrew Garfield, Eugene Otto, Dan Sharp,
+ Olivier Ruffin
+ Garry Tan, Matt Jankowski (Rails 2.2.x fixes)
+ Tim Kadom, Mauricio Marcon Zaffari, Danial Pearce, Hongli Lai, Scott Wadden (passenger fixes)
+ <your name here>
+
+Copyright (c) 2007-present Tom Anderson (tom@squeat.com), see LICENSE
View
5 vendor/plugins/spawn/init.rb
@@ -0,0 +1,5 @@
+require 'patches'
+
+ActiveRecord::Base.send :include, Spawn
+ActionController::Base.send :include, Spawn
+ActiveRecord::Observer.send :include, Spawn
View
105 vendor/plugins/spawn/lib/patches.rb
@@ -0,0 +1,105 @@
+# see activerecord/lib/active_record/connection_adaptors/abstract/connection_specification.rb
+class ActiveRecord::Base
+ # reconnect without disconnecting
+ if Spawn::RAILS_2_2
+ def self.spawn_reconnect(klass=self)
+ @@connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+ establish_connection
+ end
+ else
+ def self.spawn_reconnect(klass=self)
+ spec = @@defined_connections[klass.name]
+ konn = active_connections[klass.name]
+ # remove from internal arrays before calling establish_connection so that
+ # the connection isn't disconnected when it calls AR::Base.remove_connection
+ @@defined_connections.delete_if { |key, value| value == spec }
+ active_connections.delete_if { |key, value| value == konn }
+ establish_connection(spec ? spec.config : nil)
+ end
+ end
+
+ # this patch not needed on Rails 2.x and later
+ if Spawn::RAILS_1_x
+ # monkey patch to fix threading problems,
+ # see: http://dev.rubyonrails.org/ticket/7579
+ def self.clear_reloadable_connections!
+ if @@allow_concurrency
+ # Hash keyed by thread_id in @@active_connections. Hash of hashes.
+ @@active_connections.each do |thread_id, conns|
+ conns.each do |name, conn|
+ if conn.requires_reloading?
+ conn.disconnect!
+ @@active_connections[thread_id].delete(name)
+ end
+ end
+ end
+ else
+ # Just one level hash, no concurrency.
+ @@active_connections.each do |name, conn|
+ if conn.requires_reloading?
+ conn.disconnect!
+ @@active_connections.delete(name)
+ end
+ end
+ end
+ end
+ end
+end
+
+# see mongrel/lib/mongrel.rb
+# it's possible that this is not defined if you're running outside of mongrel
+# examples: ./script/runner or ./script/console
+if defined? Mongrel::HttpServer
+ class Mongrel::HttpServer
+ # redefine Montrel::HttpServer::process_client so that we can intercept
+ # the socket that is being used so Spawn can close it upon forking
+ alias_method :orig_process_client, :process_client
+ def process_client(client)
+ Spawn.resources_to_close(client, @socket)
+ orig_process_client(client)
+ end
+ end
+end
+
+need_passenger_patch = true
+if defined? PhusionPassenger::VERSION_STRING
+ # The VERSION_STRING variable was defined sometime after 2.1.0.
+ # We don't need passenger patch for 2.2.2 or later.
+ pv = PhusionPassenger::VERSION_STRING.split('.').collect{|s| s.to_i}
+ need_passenger_patch = pv[0] < 2 || (pv[0] == 2 && (pv[1] < 2 || (pv[1] == 2 && pv[2] < 2)))
+end
+
+if need_passenger_patch
+ # Patch for work with passenger < 2.1.0
+ if defined? Passenger::Railz::RequestHandler
+ class Passenger::Railz::RequestHandler
+ alias_method :orig_process_request, :process_request
+ def process_request(headers, input, output)
+ Spawn.resources_to_close(input, output)
+ orig_process_request(headers, input, output)
+ end
+ end
+ end
+
+ # Patch for work with passenger >= 2.1.0
+ if defined? PhusionPassenger::Railz::RequestHandler
+ class PhusionPassenger::Railz::RequestHandler
+ alias_method :orig_process_request, :process_request
+ def process_request(headers, input, output)
+ Spawn.resources_to_close(input, output)
+ orig_process_request(headers, input, output)
+ end
+ end
+ end
+
+ # Patch for passenger with Rails >= 2.3.0 (uses rack)
+ if defined? PhusionPassenger::Rack::RequestHandler
+ class PhusionPassenger::Rack::RequestHandler
+ alias_method :orig_process_request, :process_request
+ def process_request(headers, input, output)
+ Spawn.resources_to_close(input, output)
+ orig_process_request(headers, input, output)
+ end
+ end
+ end
+end
View
141 vendor/plugins/spawn/lib/spawn.rb
@@ -0,0 +1,141 @@
+module Spawn
+ RAILS_1_x = (::Rails::VERSION::MAJOR == 1) unless defined?(RAILS_1_x)
+ RAILS_2_2 = (::Rails::VERSION::MAJOR > 2 || (::Rails::VERSION::MAJOR == 2 && ::Rails::VERSION::MINOR >= 2)) unless defined?(RAILS_2_2)
+
+ # default to forking (unless windows or jruby)
+ @@method = (RUBY_PLATFORM =~ /(win32|java)/) ? :thread : :fork
+ # things to close in child process
+ @@resources = []
+ # in some environments, logger isn't defined
+ @@logger = defined?(RAILS_DEFAULT_LOGGER) ? RAILS_DEFAULT_LOGGER : Logger.new(STDERR)
+
+ # add calls to this in your environment.rb to set your configuration, for example,
+ # to use forking everywhere except your 'development' environment:
+ # Spawn::method :fork
+ # Spawn::method :thread, 'development'
+ def self.method(method, env = nil)
+ if !env || env == RAILS_ENV
+ @@method = method
+ end
+ @@logger.debug "spawn> method = #{@@method}" if defined? RAILS_DEFAULT_LOGGER
+ end
+
+ # set the resources to disconnect from in the child process (when forking)
+ def self.resources_to_close(*resources)
+ @@resources = resources
+ end
+
+ # close all the resources added by calls to resource_to_close
+ def self.close_resources
+ @@resources.each do |resource|
+ resource.close if resource && resource.respond_to?(:close) && !resource.closed?
+ end
+ # in case somebody spawns recursively
+ @@resources.clear
+ end
+
+ # Spawns a long-running section of code and returns the ID of the spawned process.
+ # By default the process will be a forked process. To use threading, pass
+ # :method => :thread or override the default behavior in the environment by setting
+ # 'Spawn::method :thread'.
+ def spawn(options = {})
+ options.symbolize_keys!
+ # setting options[:method] will override configured value in @@method
+ if options[:method] == :yield || @@method == :yield
+ yield
+ elsif options[:method] == :thread || (options[:method] == nil && @@method == :thread)
+ # for versions before 2.2, check for allow_concurrency
+ if RAILS_2_2 || ActiveRecord::Base.allow_concurrency
+ thread_it(options) { yield }
+ else
+ @@logger.error("spawn(:method=>:thread) only allowed when allow_concurrency=true")
+ raise "spawn requires config.active_record.allow_concurrency=true when used with :method=>:thread"
+ end
+ else
+ fork_it(options) { yield }
+ end
+ end
+
+ def wait(sids = [])
+ # wait for all threads and/or forks (if a single sid passed in, convert to array first)
+ Array(sids).each do |sid|
+ if sid.type == :thread
+ sid.handle.join()
+ else
+ begin
+ Process.wait(sid.handle)
+ rescue
+ # if the process is already done, ignore the error
+ end
+ end
+ end
+ # clean up connections from expired threads
+ ActiveRecord::Base.verify_active_connections!()
+ end
+
+ class SpawnId
+ attr_accessor :type
+ attr_accessor :handle
+ def initialize(t, h)
+ self.type = t
+ self.handle = h
+ end
+ end
+
+ protected
+ def fork_it(options)
+ # The problem with rails is that it only has one connection (per class),
+ # so when we fork a new process, we need to reconnect.
+ @@logger.debug "spawn> parent PID = #{Process.pid}"
+ child = fork do
+ begin
+ start = Time.now
+ @@logger.debug "spawn> child PID = #{Process.pid}"
+
+ # set the nice priority if needed
+ Process.setpriority(Process::PRIO_PROCESS, 0, options[:nice]) if options[:nice]
+
+ # disconnect from the listening socket, et al
+ Spawn.close_resources
+ # get a new connection so the parent can keep the original one
+ ActiveRecord::Base.spawn_reconnect
+
+ # run the block of code that takes so long
+ yield
+
+ rescue => ex
+ @@logger.error "spawn> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message}"
+ ensure
+ begin
+ # to be safe, catch errors on closing the connnections too
+ if RAILS_2_2
+ ActiveRecord::Base.connection_handler.clear_all_connections!
+ else
+ ActiveRecord::Base.connection.disconnect!
+ ActiveRecord::Base.remove_connection
+ end
+ ensure
+ @@logger.info "spawn> child[#{Process.pid}] took #{Time.now - start} sec"
+ # this form of exit doesn't call at_exit handlers
+ exit!(0)
+ end
+ end
+ end
+
+ # detach from child process (parent may still wait for detached process if they wish)
+ Process.detach(child)
+
+ return SpawnId.new(:fork, child)
+ end
+
+ def thread_it(options)
+ # clean up stale connections from previous threads
+ ActiveRecord::Base.verify_active_connections!()
+ thr = Thread.new do
+ # run the long-running code block
+ yield
+ end
+ return SpawnId.new(:thread, thr)
+ end
+
+end
View
76 vendor/plugins/workling/CHANGES.markdown
@@ -0,0 +1,76 @@
+Version 0.4.2.3, 31.01.2009
+- introduced Workling.raises_exceptions. by default, this is true in test and development to help with bug tracking.
+- added :threaded as the default spawn runner for test and development. helps problem tracing.
+
+Version 0.4.2.2, 29.11.08
+- turned Workling.load_path into an Array.
+
+Version 0.4.2.1, 27.11.08
+- fixed raise exceptions if non existing worker methods are called
+
+Version 0.4.2, 10.11.08
+- added information about invokers and clients to the readme
+- fixed dependence on amqp library
+- nicer error messages with amqp / rabbitmq
+
+Version 0.4.1 08.11.08
+- added a generic client runner. deprecated starling_runner since it is now redundant
+- moved connection exception handling code into MemcacheQueueClient and out of pollers
+
+Version 0.4.0, 04.11.08
+- more refactored clients and invokers. introduced clear base classes
+- support for 3 invoker strategies: basic poller, threaded poller, eventmachine subscriber
+- amqp support
+
+Version 0.3.8, 03.11.08
+- full support for rudeq
+- refactored pollers. now now longer mainly about starling
+- refactored starling client, converted to generalized memcachequeue client.
+- changed runner script to be more generic
+
+Version 0.3.1, 15.10.08
+- fixed to autodiscovery code bugs.
+- introduced Workling::VERSION
+- fixed test suite for the case that no memcache client is installed at all
+- fixed AR reconnecting code for Multicore systems (Thanks Brent)
+
+Version 0.3, 25.09.08
+- added backgroundjob runner
+- added automatic detection of starling, spawn and backgroundjob to set default runner
+- made logging of exceptions more consistent across runners.
+- added friendlier error message if starling was started on the wrong port.
+- play nice without fiveruns-memcache-client.
+- added better documentation in README and RDOC
+
+Version 0.2.5, 02.09.08
+- added automatic setting of spawn runner if the spawn plugin is installed.
+
+Version 0.2.4, 08.06.08
+- accept both async_ and asynch_ as prefixes for workling method invocation. thank you francois beausoleil!
+- added memcached configuration options to workling.yml. see example yml for details. thank you larry diehl!
+- re-raise exceptions if there is a problem adding an item to the starling queue. thank you digitalronin!
+- added status script for starling client. thank you andrew carter!
+- applied patches from dave dupre: http://davedupre.com/2008/03/29/ruby-background-tasks-with-starling-part-2/
+ - added threading to starling poller. One polling thread can now be set to run per queue.
+ - default routing no longer producing queues like a:b:c, this was conflicting with MemCacheClient#stat
+ - added handling for memcache exceptions
+ - keep the database connection alive
+
+Version 0.2.2, 15.02.08, rev 31
+- added blaine cook's suggestion: worklings can now (also) be invoked like this: YourWorkling.asynch_your_method(options)
+- added similar for remote store, which can now be called like this: Workling::Return::Store.set(:key, "value")
+
+Version 0.2.1, 14.02.08 rev. 24
+- added WorklingError classes.
+- all runners now suppresses workling exceptions. This brings the local behaviour in line with the remote runners.
+
+Version 0.2, 13.02.08 rev. 21
+- progress bars or returning results now possible with return stores. use these to communicate back from your workling.
+- memory store for testing and starling store added.
+- now generates uids for workling jobs. these are returned by the runner.
+- extracted Workling::Clients::Starling
+- clearer file structure for workling
+
+Version 0.1, 06.02.08
+- initial release
+- see http://playtype.net/past/2008/2/6/starling_and_asynchrous_tasks_in_ruby_on_rails/ for details.
View
20 vendor/plugins/workling/MIT-LICENSE
@@ -0,0 +1,20 @@
+Copyright (c) 2008 play/type GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
View
382 vendor/plugins/workling/README.markdown
@@ -0,0 +1,382 @@
+# Workling
+
+Workling gives your Rails App a simple API that you can use to make code run in the background, outside of the your request.
+
+You can configure how the background code will be run. Currently, workling supports Starling, BackgroundJob and Spawn Runners. Workling is a bit like Actve* for background work: you can write your code once, then swap in any of the supported background Runners later. This keeps things flexible.
+
+## Installing Workling
+
+The easiest way of getting started with workling is like this:
+
+ script/plugin install git://github.com/purzelrakete/workling.git
+ script/plugin install git://github.com/tra/spawn.git
+
+If you're on an older Rails version, there's also a subversion mirror wor workling (I'll do my best to keep it synched) at:
+
+ script/plugin install http://svn.playtype.net/plugins/workling/
+
+## Writing and calling Workers
+
+This is pretty easy. Just put `cow_worker.rb` into into `app/workers`, and subclass `Workling::Base`:
+
+ # handle asynchronous mooing.
+ class CowWorker < Workling::Base
+ def moo(options)
+ cow = Cow.find(options[:id])
+ logger.info("about to moo.")
+ cow.moo
+ end
+ end
+
+Make sure you have exactly one hash parameter in your methods, workling passes the job :uid into here. Btw, in case you want to follow along with the Mooing, grab 'cows-not-kittens' off github, it's an example workling project. Look at the branches, there's one for each Runner.
+
+Next, you'll want to call your workling in a controller. Your controller might looks like this:
+
+ class CowsController < ApplicationController
+
+ # milking has the side effect of causing
+ # the cow to moo. we don't want to
+ # wait for this while milking, though,
+ # it would be a terrible waste ouf our time.
+ def milk
+ @cow = Cow.find(params[:id])
+ CowWorker.asynch_moo(:id => @cow.id)
+ end
+ end
+
+Notice the `asynch_moo` call to `CowWorker`. This will call the `moo` method on the `CowWorker` in the background, passing any parameters you like on. In fact, workling will call whatever comes after asynch_ as a method on the worker instance.
+
+## Worker Lifecycle
+
+All worker classes must inherit from this class, and be saved in `app/workers`. The Worker is loaded once, at which point the instance method `create` is called.
+
+Calling `async_my_method` on the worker class will trigger background work. This means that the loaded Worker instance will receive a call to the method `my_method(:uid => "thisjobsuid2348732947923")`.
+
+## Exception handling in Workers
+
+If an exception is raised in your Worker, it will not be propagated to the calling code by workling. This is because the code is called asynchronously, meaning that exceptions may be raised after the calling code has already returned. If you need your calling code to handle exceptional situations, you have to pass the error into the return store.
+
+Workling does log all exceptions that propagate out of the worker methods.
+
+## Logging with Workling
+
+`RAILS_DEFAULT_LOGGER` is available in all workers. Workers also have a logger method which returns the default logger, so you can log like this:
+
+ logger.info("about to moo.")
+
+## What should I know about the Spawn Runner?
+
+Workling automatically detects and uses Spawn, if installed. Spawn basically forks Rails every time you invoke a workling. To see what sort of characteristics this has, go into script/console, and run this:
+
+ >> fork { sleep 100 }
+ => 1060 (the pid is returned)
+
+You'll see that this executes pretty much instantly. Run 'top' in another terminal window, and look for the new ruby process. This might be around 30 MB. This tells you that using spawn as a runner will result low latency, but will take at least 30MB for each request you make.
+
+You cannot run your workers on a remote machine or cluster them with spawn. You also have no persistence: if you've fired of a lot of work and everything dies, there's no way of picking up where you left off.
+
+# Using the Starling runner
+
+If you want cross machine jobs with low latency and a low memory overhead, you might want to look into using the Starling Runner.
+
+## Installing Starling
+
+As of 27. September 2008, the recommended Starling setup is as follows:
+
+ gem sources -a http://gems.github.com/
+ sudo gem install starling-starling
+ mkdir /var/spool/starling
+
+The robot Co-Op Memcached Gem version 1.5.0 has several bugs, which have been fixed in the fiveruns-memcache-client gem. The starling-starling gem will install this as a dependency. Refer to the fiveruns README to see what the exact fixes are.
+
+The Rubyforge Starling gem is also out of date. Currently, the most authorative Project is starling-starling on github (27. September 2008).
+
+Workling will now automatically detect and use Starling, unless you have also installed Spawn. If you have Spawn installed, you need to tell Workling to use Starling by putting this in your environment.rb:
+
+ Workling::Remote.dispatcher = Workling::Remote::Runners::StarlingRunner.new
+
+## Starting up the required processes
+
+Here's what you need to get up and started in development mode. Look in config/workling.yml to see what the default ports are for other environments.
+
+ sudo starling -d -p 22122
+ script/workling_client start
+
+## Configuring workling.yml
+
+Workling copies a file called workling.yml into your applications config directory. The config file tells Workling on which port Starling is listening.
+
+Notice that the default production port is 15151. This means you'll need to start Starling with -p 15151 on production.
+
+You can also use this config file to pass configuration options to the memcache client which workling uses to connect to starling. use the key 'memcache_options' for this.
+
+You can also set sleep time for each Worker. See the key 'listeners' for this. Put in the modularized Class name as a key.
+
+ development:
+ listens_on: localhost:22122
+ sleep_time: 2
+ reset_time: 30
+ listeners:
+ Util:
+ sleep_time: 20
+ memcache_options:
+ namespace: myapp_development
+
+ production:
+ listens_on: localhost:22122, localhost:221223, localhost:221224
+ sleep_time: 2
+ reset_time: 30
+
+Note that you can cluster Starling instances by passing a comma separated list of values to
+
+Sleep time determines the wait time between polls against polls. A single poll will do one .get on every queue (there is a corresponding queue for each worker method).
+
+If there is a memcache error, the Poller will hang for a bit to give it a chance to fire up again and reset the connection. The wait time can be set with the key reset_time.
+
+## Seeing what Starling is doing
+
+Starling comes with it's own script, starling_top. If you want statistics specific to workling, run:
+
+ script/starling_status.rb
+
+## A Quick Starling Primer
+
+You might wonder what exactly starling does. Here's a little snippet you can play with to illustrate how it works:
+
+ 4 # Put messages onto a queue:
+ 5 require 'memcache'
+ 6 starling = MemCache.new('localhost:22122')
+ 7 starling.set('my_queue', 1)
+ 8
+ 9 # Get messages from the queue:
+ 10 require 'memcache'
+ 11 starling = MemCache.new('localhost:22122')
+ 12 loop { puts starling.get('my_queue') }
+ 13
+
+# Using RabbitMQ or any Queue Server that supports AMQP
+
+RabbitMQ is a reliable, high performance queue server written in erlang. If you're doing high volume messaging and need a high degree of reliability, you should definitely consider using RabbitMQ over Starling.
+
+A lot of Ruby people have been talking about using RabbitMQ as their Queue of choice. Soundcloud.com are using it, as is new bamboo founder Johnathan Conway, who is using it at his video startup http://www.vzaar.com/. He says:
+
+> RabbitMQ – Now this is the matrons knockers when it comes to kick ass, ultra fast and scalable messaging. It simply rocks, with performance off the hook. It’s written in Erlang and supports the AMPQ protocol.
+
+If you're on OSX, you can get started with RabbitMQ by following the installation instructions [here](http://playtype.net/past/2008/10/9/installing_rabbitmq_on_osx/). To get an idea of how to directly connect to RabbitMQ using ruby, have a look at [this article](http://playtype.net/past/2008/10/10/kickass_queuing_over_ruby_using_amqp).
+
+Once you've installed RabbitMQ, install the ruby amqp library:
+
+ gem sources -a http://gems.github.com/ (if necessary)
+ sudo gem install tmm1-amqp
+
+then configure configure your application to use Amqp by adding this:
+
+ Workling::Remote.invoker = Workling::Remote::Invokers::EventmachineSubscriber
+ Workling::Remote.dispatcher = Workling::Remote::Runners::ClientRunner.new
+ Workling::Remote.dispatcher.client = Workling::Clients::AmqpClient.new
+
+Then start the workling Client:
+
+ 1 ./script/workling_client start
+
+You're good.
+
+# Using RudeQueue
+
+RudeQueue is a Starling-like Queue that runs on top of your database and requires no extra processes. Use this if you don't need very fast job processing and want to avoid managing the extra process starling requires.
+
+Install the RudeQ plugin like this:
+
+ 1 ./script/plugin install git://github.com/matthewrudy/rudeq.git
+ 2 rake queue:setup
+ 3 rake db:migrate
+
+Configure Workling to use RudeQ. Add this to your environment:
+
+ Workling::Clients::MemcacheQueueClient.memcache_client_class = RudeQ::Client
+ Workling::Remote.dispatcher = Workling::Remote::Runners::ClientRunner.new
+
+Now start the Workling Client:
+
+ 1 ./script/workling_client start
+
+You're good.
+
+# Using BackgroundJob
+
+If you don't want to bother with seperate processes, are not worried about latence or memory footprint, then you might want to use Bj to power workling.
+
+Install the Bj plugin like this:
+
+ 1 ./script/plugin install http://codeforpeople.rubyforge.org/svn/rails/plugins/bj
+ 2 ./script/bj setup
+
+Workling will now automatically detect and use Bj, unless you have also installed Starling. If you have Starling installed, you need to tell Workling to use Bj by putting this in your environment.rb:
+
+ Workling::Remote.dispatcher = Workling::Remote::Runners::BackgroundjobRunner.new
+
+# Progress indicators and return stores
+
+Your worklings can write back to a return store. This allows you to write progress indicators, or access results from your workling. As above, this is fairly slim. Again, you can swap in any return store implementation you like without changing your code. They all behave like memcached. For tests, there is a memory return store, for production use there is currently a starling return store. You can easily add a new return store (over the database for instance) by subclassing `Workling::Return::Store::Base`. Configure it like this in your test environment:
+
+ Workling::Return::Store.instance = Workling::Return::Store::MemoryReturnStore.new
+
+Setting and getting values works as follows. Read the next paragraph to see where the job-id comes from.
+
+ Workling.return.set("job-id-1", "moo")
+ Workling.return.get("job-id-1") => "moo"
+
+Here is an example worker that crawls an addressbook and puts results into a return store. Workling makes sure you have a :uid in your argument hash - set the value into the return store using this uid as a key:
+
+ require 'blackbook'
+ class NetworkWorker < Workling::Base
+ def search(options)
+ results = Blackbook.get(options[:key], options[:username], options[:password])
+ Workling.return.set(options[:uid], results)
+ end
+ end
+
+call your workling as above:
+
+ @uid = NetworkWorker.asynch_search(:key => :gmail, :username => "foo@gmail.com", :password => "bar")
+
+you can now use the @uid to query the return store:
+
+ results = Workling.return.get(@uid)
+
+of course, you can use this for progress indicators. just put the progress into the return store.
+
+enjoy!
+
+## Adding new work brokers to Workling
+
+There are two new base classes you can extend to add new brokers. I'll describe how this is done usin amqp as an example. The code i show is already a part of workling.
+
+### Clients
+
+Clients help workling to connect to job brokers. To add an AmqpClient, we need to extend from `Workling::Client::Base` and implement a couple of methods.
+
+ require 'workling/clients/base'
+ require 'mq'
+
+ #
+ # An Ampq client
+ #
+ module Workling
+ module Clients
+ class AmqpClient < Workling::Clients::Base
+
+ # starts the client.
+ def connect
+ @amq = MQ.new
+ end
+
+ # stops the client.
+ def close
+ @amq.close
+ end
+
+ # request work
+ def request(queue, value)
+ @amq.queue(queue).publish(value)
+ end
+
+ # retrieve work
+ def retrieve(queue)
+ @amq.queue(queue)
+ end
+
+ # subscribe to a queue
+ def subscribe(queue)
+ @amq.queue(queue).subscribe do |value|
+ yield value
+ end
+ end
+
+ end
+ end
+ end
+
+Were's using the eventmachine amqp client for this, you can find it [up on github](http://github.com/tmm1/amqp/tree/master). `connect` and `close` do exactly what it says on the tin: connecting to rabbitmq and closing the connection.
+
+`request` and `retrieve` are responsible for placing work on rabbitmq. The methods are passed the correct queue, and a value that contains the worker method arguments. If you need control over the queue names, look at the RDoc for Workling::Routing::Base. In our case, there's no special requirement here.
+
+Finally, we implement a `subscribe` method. Use this if your broker supports callbacks, as is the case with amqp. This method expects to a block, which we pass into the amqp subscribe method here. The block will be called when a message is available on the queue, and the result is yielded into the block.
+
+Having subscription callbacks is very nice, because this way, we don't need to keep calling `get` on the queue to see if something new is waiting.
+
+So now we're done! That's all you need to add RabbitMQ to workling. Configure it in your application as descibed below.
+
+### Invokers
+
+There's still potential to improve things though. Workling 0.4.0 introduces the idea of invokers. Invokers grab work off a job broker, using a client (see above). They subclass Workling::Remote::Invokers::Base. Read the RDoc for a description of the methods.
+
+Workling comes with a couple of standard invokers, like the BasicPoller. This invoker simply keeps hitting the broker every n seconds, checking for new work and executing it immediately. The ThreadedInvoker does the same, but spawns a Thread for every Worker class the project defines.
+
+So Amqp: it would be nice if we had an invoker that makes use of the subscription callbacks. Easily done, lets have a look:
+
+ require 'eventmachine'
+ require 'workling/remote/invokers/base'
+
+ #
+ # Subscribes the workers to the correct queues.
+ #
+ module Workling
+ module Remote
+ module Invokers
+ class EventmachineSubscriber < Workling::Remote::Invokers::Base
+
+ def initialize(routing, client_class)
+ super
+ end
+
+ #
+ # Starts EM loop and sets up subscription callbacks for workers.
+ #
+ def listen
+ EM.run do
+ connect do
+ routes.each do |queue|
+ @client.subscribe(queue) do |args|
+ run(queue, args)
+ end
+ end
+ end
+ end
+ end
+
+ def stop
+ EM.stop if EM.reactor_running?
+ end
+ end
+ end
+ end
+ end
+
+Invokers have to implement two methods, `listen` and `stop`. Listen starts the main listener loop, which is responsible for starting work when it becomes available.
+
+In our case, we need to start an EM loop around `listen`. This is because the Ruby AMQP library needs to run inside of an eventmachine reactor loop.
+
+Next, inside of `listen`, we need to iterate through all defined routes. There is a route for each worker method you defined in your application. The routes double as queue names. For this, you can use the helper method `routes`. Now we attach a callback to each queue. We can use the helper method `run`, which executes the worker method associated with the queue, passing along any supplied arguments.
+
+That's it! We now have a more effective Invoker.
+
+# Contributors
+
+The following people contributed code to workling so far. Many thanks :) If I forgot anybody, I aplogise. Just drop me a note and I'll add you to the project so that you can amend this!
+
+Anybody who contributes fixes (with tests), or new functionality (whith tests) which is pulled into the main project, will also be be added to the project.
+
+* Andrew Carter (ascarter)
+* Chris Gaffney (gaffneyc)
+* Matthew Rudy (matthewrudy)
+* Larry Diehl (reeze)
+* grantr (francios)
+* David (digitalronin)
+* Dave Dupré
+* Douglas Shearer (dougal)
+* Nick Plante (zapnap)
+* Brent
+* Evan Light (elight)
+
+Copyright (c) 2008 play/type GmbH, released under the MIT license
View
22 vendor/plugins/workling/Rakefile
@@ -0,0 +1,22 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the loopy_workling plugin.'
+Rake::TestTask.new(:test) do |t|
+ t.libs << 'lib'
+ t.pattern = 'test/**/*_test.rb'
+ t.verbose = true
+end
+
+desc 'Generate documentation for the loopy_workling plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+ rdoc.rdoc_dir = 'rdoc'
+ rdoc.title = 'LoopyWorkling'
+ rdoc.options << '--line-numbers' << '--inline-source'
+ rdoc.rdoc_files.include('README.markdown')
+ rdoc.rdoc_files.include('lib/**/*.rb')
+end
View
26 vendor/plugins/workling/TODO.markdown
@@ -0,0 +1,26 @@
+# Todos for 0.5.0
+
+* add a linting runner for tests. should check that no ar objects are being passed around
+* add a mechanism for requiring models, for those people who insist on passing models across the wire
+* add reloading of workers if Rails.reload?
+* write some code that knows if the client should be started, and gives out a warning
+* add a configuration option for SERVER/CLIENT
+* add phusion daemon starter option so that workling_client doesn't need to be started manually on SERVER
+* write some more documentation on the above issues and on standard remote setup.
+* create a public forum, rdoc site
+* try to reduce user error in setting environments correctly
+* add beanstalkd runner
+* refactor starling* to be memcache*. add aliased classes into deprecated.rb.
+* look into create method. is this being called more often than intended?
+* add some monit and god scripts as starters
+* try to catch more user setup errors which lead to worker code not being called
+
+# Todos for 1.0
+
+* gemify
+* move all runner/invoker implementations out of workling
+* move backend discovery code out of workling
+* decide on a single backend to include in workling
+* merb support
+* test on jruby
+* more runners: sqs
View
16 vendor/plugins/workling/config/workling.yml
@@ -0,0 +1,16 @@
+# By default, NotRemoteRunner is used when RAILS_ENV == 'test'.
+#
+# You can pass options to memcached client by nesting the key value pairs
+# under 'memcache_options'.
+#
+# You can also use a cluster of Starlings. Simply give a comma separated
+# list of server:port, server:port, server:port values to listens_on.
+#
+production:
+ listens_on: localhost:15151
+
+development:
+ listens_on: localhost:22122
+
+test:
+ listens_on: localhost:12345
View
3 vendor/plugins/workling/init.rb
@@ -0,0 +1,3 @@
+Workling.try_load_a_memcache_client
+Workling::Discovery.discover!
+Workling.config
View
17 vendor/plugins/workling/install.rb
@@ -0,0 +1,17 @@
+require 'fileutils'
+
+plugin_root = File.dirname(__FILE__)
+
+# config files
+for file in config = %w{ workling.yml } do
+ FileUtils.cp(File.join(plugin_root, 'config', file), File.join(RAILS_ROOT, 'config'))
+end
+
+# scripts
+for file in script = %w{ workling_starling_client workling_client bj_invoker.rb starling_status.rb } do
+ FileUtils.cp File.join(plugin_root, 'script', file), File.join(RAILS_ROOT, 'script')
+ FileUtils.chmod 0755, File.join(RAILS_ROOT, 'script', file)
+end
+
+puts "\n\ninstalled #{ (script + config).join(", ") } \n\n"
+puts IO.read(File.join(File.dirname(__FILE__), 'README.markdown'))
View
11 vendor/plugins/workling/lib/rude_q/client.rb
@@ -0,0 +1,11 @@
+#
+# A RudeQ client that behvaes somewhat like memcache-client
+#
+module RudeQ
+ class Client
+ def initialize(*args); super(); end
+ def set(key, value); RudeQueue.set(key, value); end;
+ def get(key); RudeQueue.get(key); end;
+ def stats; ActiveRecord::Base.connection; end
+ end
+end
View
150 vendor/plugins/workling/lib/workling.rb
@@ -0,0 +1,150 @@
+#
+# I can haz am in your Workling are belong to us!
+#
+module Workling
+ class WorklingError < StandardError; end
+ class WorklingNotFoundError < WorklingError; end
+ class WorklingConnectionError < WorklingError; end
+ class QueueserverNotFoundError < WorklingError
+ def initialize
+ super "config/workling.yml configured to connect to queue server on #{ Workling.config[:listens_on] } for this environment. could not connect to queue server on this host:port. for starling users: pass starling the port with -p flag when starting it. If you don't want to use Starling, then explicitly set Workling::Remote.dispatcher (see README for an example)"
+ end
+ end
+
+ class ConfigurationError < WorklingError
+ def initialize
+ super File.exist?(File.join(RAILS_ROOT, 'config', 'starling.yml')) ?
+ "config/starling.yml has been depracated. rename your config file to config/workling.yml then try again!" :
+ "config/workling.yml could not be loaded. check out README.markdown to see what this file should contain. "
+ end
+ end
+
+ mattr_accessor :load_path
+ @@load_path = [ File.expand_path(File.join(File.dirname(__FILE__), '../../../../app/workers')) ]
+ VERSION = "0.4.2.3"
+
+ #
+ # determine the runner to use if nothing is specifically set. workling will try to detect
+ # starling, spawn, or bj, in that order. if none of these are found, notremoterunner will
+ # be used.
+ #
+ # this can be overridden by setting Workling::Remote.dispatcher, eg:
+ # Workling::Remote.dispatcher = Workling::Remote::Runners::StarlingRunner.new
+ #
+ def self.default_runner
+ if RAILS_ENV == "test"
+ Workling::Remote::Runners::NotRemoteRunner.new
+ elsif starling_installed?
+ Workling::Remote::Runners::StarlingRunner.new
+ elsif spawn_installed?
+ Workling::Remote::Runners::SpawnRunner.new
+ elsif bj_installed?
+ Workling::Remote::Runners::BackgroundjobRunner.new
+ else
+ Workling::Remote::Runners::NotRemoteRunner.new
+ end
+ end
+
+ #
+ # gets the worker instance, given a class. the optional method argument will cause an
+ # exception to be raised if the worker instance does not respoind to said method.
+ #
+ def self.find(clazz, method = nil)
+ begin
+ inst = clazz.to_s.camelize.constantize.new
+ rescue NameError
+ raise_not_found(clazz, method)
+ end
+ raise_not_found(clazz, method) if method && !inst.respond_to?(method)
+ inst
+ end
+
+ # returns Workling::Return::Store.instance.
+ def self.return
+ Workling::Return::Store.instance
+ end
+
+ # is spawn installed?
+ def self.spawn_installed?
+ begin
+ require 'spawn'
+ rescue LoadError
+ end
+
+ Object.const_defined? "Spawn"
+ end
+
+ # is starling installed?
+ def self.starling_installed?
+ begin
+ require 'starling'
+ rescue LoadError
+ end
+
+ Object.const_defined? "Starling"
+ end
+
+ # is bj installed?
+ def self.bj_installed?
+ Object.const_defined? "Bj"
+ end
+
+ # tries to load fiveruns-memcache-client. if this isn't found,
+ # memcache-client is searched for. if that isn't found, don't do anything.
+ def self.try_load_a_memcache_client
+ begin
+ gem 'fiveruns-memcache-client'
+ require 'memcache'
+ rescue Gem::LoadError
+ begin
+ gem 'memcache-client'
+ require 'memcache'
+ rescue Gem::LoadError
+ Workling::Base.logger.info "WORKLING: couldn't find a memcache client - you need one for the starling runner. "
+ end
+ end
+ end
+
+ # attempts to load amqp and writes out descriptive error message if not present
+ def self.try_load_an_amqp_client
+ begin
+ require 'mq'
+ rescue Exception => e
+ raise WorklingError.new(
+ "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
+ "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
+ )
+ end
+ end
+
+ #
+ # returns a config hash. reads RAILS_ROOT/config/workling.yml
+ #
+ def self.config
+ begin
+ config_path = File.join(RAILS_ROOT, 'config', 'workling.yml')
+ @@config ||= YAML.load_file(config_path)[RAILS_ENV || 'development'].symbolize_keys
+ @@config[:memcache_options].symbolize_keys! if @@config[:memcache_options]
+ @@config
+ rescue
+ # config files could not be read correctly
+ raise ConfigurationError.new
+ end
+ end
+
+ #
+ # Raises exceptions thrown inside of the worker. normally, these are logged to
+ # logger.error. it's easy to miss these log calls while developing, though.
+ #
+ mattr_accessor :raise_exceptions
+ @@raise_exceptions = (RAILS_ENV == "test" || RAILS_ENV == "development")
+
+ def self.raise_exceptions?
+ @@raise_exceptions
+ end
+
+ private
+ def self.raise_not_found(clazz, method)
+ raise Workling::WorklingNotFoundError.new("could not find #{ clazz }:#{ method } workling. ")
+ end
+end
View
59 vendor/plugins/workling/lib/workling/base.rb
@@ -0,0 +1,59 @@
+#
+# All worker classes must inherit from this class, and be saved in app/workers.
+#
+# The Worker lifecycle:
+# The Worker is loaded once, at which point the instance method 'create' is called.
+#
+# Invoking Workers:
+# Calling async_my_method on the worker class will trigger background work.
+# This means that the loaded Worker instance will receive a call to the method
+# my_method(:uid => "thisjobsuid2348732947923").
+#
+# The Worker method must have a single hash argument. Note that the job :uid will
+# be merged into the hash.
+#
+module Workling
+ class Base
+ cattr_accessor :logger
+ @@logger ||= ::RAILS_DEFAULT_LOGGER
+
+ def self.inherited(subclass)
+ Workling::Discovery.discovered << subclass
+ end
+
+ def initialize
+ super
+
+ create
+ end
+
+ # Put worker initialization code in here. This is good for restarting jobs that
+ # were interrupted.
+ def create
+ end
+
+ # takes care of suppressing remote errors but raising Workling::WorklingNotFoundError
+ # where appropriate. swallow workling exceptions so that everything behaves like remote code.
+ # otherwise StarlingRunner and SpawnRunner would behave too differently to NotRemoteRunner.
+ def dispatch_to_worker_method(method, options)
+ begin
+ self.send(method, options)
+ rescue Exception => e
+ raise e if e.kind_of?(Workling::WorklingError)
+ logger.error "WORKLING ERROR: runner could not invoke #{ self.class }:#{ method } with #{ options.inspect }. error was: #{ e.inspect }\n #{ e.backtrace.join("\n") }"
+
+ # reraise after logging. the exception really can't go anywhere in many cases. (spawn traps the exception)
+ raise e if Workling.raise_exceptions?
+ end
+ end
+
+ # thanks to blaine cook for this suggestion.
+ def self.method_missing(method, *args, &block)
+ if method.to_s =~ /^asynch?_(.*)/
+ Workling::Remote.run(self.to_s.dasherize, $1, *args)
+ else
+ super
+ end
+ end
+ end
+end
View
40 vendor/plugins/workling/lib/workling/clients/amqp_client.rb
@@ -0,0 +1,40 @@
+require 'workling/clients/base'
+Workling.try_load_an_amqp_client
+
+#
+# An Ampq client
+#
+module Workling
+ module Clients
+ class AmqpClient < Workling::Clients::Base
+
+ # starts the client.
+ def connect
+ begin
+ @amq = MQ.new
+ rescue
+ raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
+ end
+ end
+
+ # no need for explicit closing. when the event loop
+ # terminates, the connection is closed anyway.
+ def close; true; end
+
+ # subscribe to a queue
+ def subscribe(key)
+ @amq.queue(key).subscribe do |data|
+ value = Marshal.load(data)
+ yield value
+ end
+ end
+
+ # request and retrieve work
+ def retrieve(key); @amq.queue(key); end
+ def request(key, value)
+ data = Marshal.dump(value)
+ @amq.queue(key).publish(data)
+ end
+ end
+ end
+end
View
54 vendor/plugins/workling/lib/workling/clients/base.rb
@@ -0,0 +1,54 @@
+#
+# Clients are responsible for communicating with a job broker (ie connecting to starling or rabbitmq.)
+#
+# Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
+# from a specific type of job.
+#
+module Workling
+ module Clients
+ class Base
+
+ #
+ # Requests a job on the broker.
+ #
+ # work_type:
+ # arguments: the argument to the worker method
+ #
+ def request(work_type, arguments)
+ raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
+ end
+
+ #
+ # Gets job results off a job broker. Returns nil if there are no results.
+ #
+ # worker_uid: the uid returned by workling when the work was dispatched
+ #
+ def retrieve(work_uid)
+ raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
+ end
+
+ #
+ # Subscribe to job results in a job broker.
+ #
+ # worker_type:
+ #
+ def subscribe(work_type)
+ raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
+ end
+
+ #
+ # Opens a connection to the job broker.
+ #
+ def connect
+ raise NotImplementedError.new("Implement connect() in your client. ")
+ end
+
+ #
+ # Closes the connection to the job broker.
+ #
+ def close
+ raise NotImplementedError.new("Implement close() in your client. ")
+ end
+ end
+ end
+end
View
82 vendor/plugins/workling/lib/workling/clients/memcache_queue_client.rb
@@ -0,0 +1,82 @@
+require 'workling/clients/base'
+
+#
+# This client can be used for all Queue Servers that speak Memcached, such as Starling.
+#
+# Wrapper for the memcache connection. The connection is made using fiveruns-memcache-client,
+# or memcache-client, if this is not available. See the README for a discussion of the memcache
+# clients.
+#
+# method_missing delegates all messages through to the underlying memcache connection.
+#
+module Workling
+ module Clients
+ class MemcacheQueueClient < Workling::Clients::Base
+
+ # the class with which the connection is instantiated
+ cattr_accessor :memcache_client_class
+ @@memcache_client_class ||= ::MemCache
+
+ # the url with which the memcache client expects to reach starling
+ attr_accessor :queueserver_urls
+
+ # the memcache connection object
+ attr_accessor :connection
+
+ #
+ # the client attempts to connect to queueserver using the configuration options found in
+ #
+ # Workling.config. this can be configured in config/workling.yml.
+ #
+ # the initialization code will raise an exception if memcache-client cannot connect
+ # to queueserver.
+ #
+ def connect
+ @queueserver_urls = Workling.config[:listens_on].split(',').map { |url| url ? url.strip : url }
+ options = [@queueserver_urls, Workling.config[:memcache_options]].compact
+ self.connection = MemcacheQueueClient.memcache_client_class.new(*options)
+
+ raise_unless_connected!
+ end
+
+ # closes the memcache connection
+ def close
+ self.connection.flush_all
+ self.connection.reset
+ end
+
+ # implements the client job request and retrieval
+ def request(key, value)
+ set(key, value)
+ end
+
+ def retrieve(key)
+ begin
+ get(key)
+ rescue MemCache::MemCacheError => e
+ # failed to enqueue, raise a workling error so that it propagates upwards
+ raise Workling::WorklingError.new("#{e.class.to_s} - #{e.message}")
+ end
+ end
+
+ private
+ # make sure we can actually connect to queueserver on the given port
+ def raise_unless_connected!
+ begin
+ self.connection.stats
+ rescue
+ raise Workling::QueueserverNotFoundError.new
+ end
+ end
+
+ # delegates directly through to the memcache connection.
+ def method_missing(method, *args)
+ begin
+ self.connection.send(method, *args)
+ rescue MemCache::MemCacheError => e
+ raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")
+ end
+ end
+ end
+ end
+end
View
14 vendor/plugins/workling/lib/workling/discovery.rb
@@ -0,0 +1,14 @@
+#
+# Discovery is responsible for loading workers in app/workers.
+#
+module Workling
+ class Discovery
+ cattr_accessor :discovered
+ @@discovered = []
+
+ # requires worklings so that they are added to routing.
+ def self.discover!
+ Dir.glob(Workling.load_path.map { |p| "#{ p }/**/*.rb" }).each { |wling| require wling }
+ end
+ end
+end
View
42 vendor/plugins/workling/lib/workling/remote.rb
@@ -0,0 +1,42 @@
+require "workling/remote/runners/not_remote_runner"
+require "workling/remote/runners/spawn_runner"
+require "workling/remote/runners/starling_runner"
+require "workling/remote/runners/backgroundjob_runner"
+
+require 'digest/md5'
+
+#
+# Scoping Module for Runners.
+#
+module Workling
+ module Remote
+
+ # set the desired runner here. this is initialized with Workling.default_runner.
+ mattr_accessor :dispatcher
+
+ # set the desired invoker. this class grabs work from the job broker and executes it.
+ mattr_accessor :invoker
+ @@invoker ||= Workling::Remote::Invokers::ThreadedPoller
+
+ # retrieve the dispatcher or instantiate it using the defaults
+ def self.dispatcher
+ @@dispatcher ||= Workling.default_runner
+ end
+
+ # generates a unique identifier for this particular job.
+ def self.generate_uid(clazz, method)
+ uid = ::Digest::MD5.hexdigest("#{ clazz }:#{ method }:#{ rand(1 << 64) }:#{ Time.now }")
+ "#{ clazz.to_s.tableize }/#{ method }/#{ uid }".split("/").join(":")
+ end
+
+ # dispatches to a workling. writes the :uid for this work into the options hash, so make
+ # sure you pass in a hash if you want write to a return store in your workling.
+ def self.run(clazz, method, options = {})
+ uid = Workling::Remote.generate_uid(clazz, method)
+ options[:uid] = uid if options.kind_of?(Hash) && !options[:uid]
+ Workling.find(clazz, method) # this line raises a WorklingError if the method does not exist.
+ dispatcher.run(clazz, method, options)
+ uid
+ end
+ end
+end
View
124 vendor/plugins/workling/lib/workling/remote/invokers/base.rb
@@ -0,0 +1,124 @@
+#
+# Invokers are responsible for
+#
+# 1. grabbing work off a job broker (such as a starling or rabbitmq server).
+# 2. routing (mapping) that work onto the correct worker method.
+# 3.invoking the worker method, passing any arguments that came off the broker.
+#
+# Invokers should implement their own concurrency strategies. For example,
+# The there is a ThreadedPoller which starts a thread for each Worker class.
+#
+# This base Invoker class defines the methods an Invoker needs to implement.
+#
+module Workling
+ module Remote
+ module Invokers
+ class Base
+
+ attr_accessor :sleep_time, :reset_time
+
+ #
+ # call up with super in the subclass constructor.
+ #
+ def initialize(routing, client_class)
+ @routing = routing
+ @client_class = client_class
+ @sleep_time = Workling.config[:sleep_time] || 2
+ @reset_time = Workling.config[:reset_time] || 30
+ @@mutex ||= Mutex.new
+ end
+
+ #
+ # Starts main Invoker Loop. The invoker runs until stop() is called.
+ #
+ def listen
+ raise NotImplementedError.new("Implement listen() in your Invoker. ")
+ end
+
+ #
+ # Gracefully stops the Invoker. The currently executing Jobs should be allowed
+ # to finish.
+ #
+ def stop
+ raise NotImplementedError.new("Implement stop() in your Invoker. ")
+ end
+
+ #
+ # Runs the worker method, given
+ #
+ # type: the worker route
+ # args: the arguments to be passed into the worker method.
+ #
+ def run(type, args)
+ worker = @routing[type]
+ method = @routing.method_name(type)
+ worker.dispatch_to_worker_method(method, args)
+ end
+
+ # returns the Workling::Base.logger
+ def logger; Workling::Base.logger; end
+
+ protected
+
+ # handle opening and closing of client. pass code block to this method.
+ def connect
+ @client = @client_class.new
+ @client.connect
+
+ begin
+ yield
+ ensure
+ @client.close
+ ActiveRecord::Base.verify_active_connections!
+ end
+ end
+
+ #
+ # Loops through the available routes, yielding for each route.
+ # This continues until @shutdown is set on this instance.
+ #
+ def loop_routes
+ while(!@shutdown) do
+ ensure_activerecord_connection
+
+ routes.each do |route|
+ break if @shutdown
+ yield route
+ end
+
+ sleep self.sleep_time
+ end
+ end
+
+ #
+ # Returns the complete set of active routes
+ #
+ def routes
+ @active_routes ||= Workling::Discovery.discovered.map { |clazz| @routing.queue_names_routing_class(clazz) }.flatten
+ end
+
+ # Thanks for this Brent!
+ #
+ # ...Just a heads up, due to how rails’ MySQL adapter handles this
+ # call ‘ActiveRecord::Base.connection.active?’, you’ll need
+ # to wrap the code that checks for a connection in in a mutex.
+ #
+ # ....I noticed this while working with a multi-core machine that
+ # was spawning multiple workling threads. Some of my workling
+ # threads would hit serious issues at this block of code without
+ # the mutex.
+ #
+ def ensure_activerecord_connection
+ @@mutex.synchronize do
+ unless ActiveRecord::Base.connection.active? # Keep MySQL connection alive
+ unless ActiveRecord::Base.connection.reconnect!
+ logger.fatal("Failed - Database not available!")
+ break
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
View
41 vendor/plugins/workling/lib/workling/remote/invokers/basic_poller.rb
@@ -0,0 +1,41 @@
+require 'workling/remote/invokers/base'
+
+#
+# A basic polling invoker.
+#
+module Workling
+ module Remote
+ module Invokers
+ class BasicPoller < Workling::Remote::Invokers::Base
+
+ #
+ # set up client, sleep time
+ #
+ def initialize(routing, client_class)
+ super
+ end
+
+ #
+ # Starts main Invoker Loop. The invoker runs until stop() is called.
+ #
+ def listen
+ connect do
+ loop_routes do |route|
+ if args = @client.retrieve(route)
+ run(route, args)
+ end
+ end
+ end
+ end
+
+ #
+ # Gracefully stops the Invoker. The currently executing Jobs should be allowed
+ # to finish.
+ #
+ def stop
+ @shutdown = true
+ end
+ end
+ end
+ end
+end
View
37 vendor/plugins/workling/lib/workling/remote/invokers/eventmachine_subscriber.rb
@@ -0,0 +1,37 @@
+require 'eventmachine'
+require 'workling/remote/invokers/base'
+
+#
+# Subscribes the workers to the correct queues.
+#
+module Workling
+ module Remote
+ module Invokers
+ class EventmachineSubscriber < Workling::Remote::Invokers::Base
+
+ def initialize(routing, client_class)
+ super
+ end
+
+ #
+ # Starts EM loop and sets up subscription callbacks for workers.
+ #
+ def listen
+ EM.run do
+ connect do
+ routes.each do |route|
+ @client.subscribe(route) do |args|
+ run(route, args)
+ end
+ end
+ end
+ end
+ end
+
+ def stop
+ EM.stop if EM.reactor_running?
+ end
+ end
+ end
+ end
+end
View
150 vendor/plugins/workling/lib/workling/remote/invokers/threaded_poller.rb
@@ -0,0 +1,150 @@
+require 'workling/remote/invokers/base'
+
+#
+# A threaded polling Invoker.
+#
+# TODO: refactor this to make use of the base class.
+#
+module Workling
+ module Remote
+ module Invokers
+ class ThreadedPoller < Workling::Remote::Invokers::Base
+
+ cattr_accessor :sleep_time, :reset_time
+
+ def initialize(routing, client_class)
+ super
+
+ ThreadedPoller.sleep_time = Workling.config[:sleep_time] || 2
+ ThreadedPoller.reset_time = Workling.config[:reset_time] || 30
+
+ @workers = ThreadGroup.new
+ @mutex = Mutex.new
+ end
+
+ def listen
+ # Allow concurrency for our tasks
+ ActiveRecord::Base.allow_concurrency = true
+
+ # Create a thread for each worker.
+ Workling::Discovery.discovered.each do |clazz|
+ logger.debug("Discovered listener #{clazz}")
+ @workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
+ end
+
+ # Wait for all workers to complete
+ @workers.list.each { |t| t.join }
+
+ logger.debug("Reaped listener threads. ")
+
+ # Clean up all the connections.
+ ActiveRecord::Base.verify_active_connections!
+ logger.debug("Cleaned up connection: out!")
+ end
+
+ # Check if all Worker threads have been started.
+ def started?
+ logger.debug("checking if started... list size is #{ worker_threads }")
+ Workling::Discovery.discovered.size == worker_threads
+ end
+
+ # number of worker threads running
+ def worker_threads
+ @workers.list.size
+ end
+
+ # Gracefully stop processing
+ def stop
+ logger.info("stopping threaded poller...")
+ sleep 1 until started? # give it a chance to start up before shutting down.
+ logger.info("Giving Listener Threads a chance to shut down. This may take a while... ")
+ @workers.list.each { |w| w[:shutdown] = true }
+ logger.info("Listener threads were shut down. ")
+ end
+
+ # Listen for one worker class
+ def clazz_listen(clazz)
+ logger.debug("Listener thread #{clazz.name} started")
+
+ # Read thread configuration if available
+ if Workling.config.has_key?(:listeners)
+ if Workling.config[:listeners].has_key?(clazz.to_s)
+ config = Workling.config[:listeners][clazz.to_s].symbolize_keys
+ thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
+ end
+ end
+
+ hread_sleep_time ||= self.class.sleep_time
+
+ # Setup connection to client (one per thread)
+ connection = @client_class.new
+ connection.connect
+ logger.info("** Starting client #{ connection.class } for #{clazz.name} queue")
+
+ # Start dispatching those messages
+ while (!Thread.current[:shutdown]) do
+ begin
+
+ # Thanks for this Brent!
+ #
+ # ...Just a heads up, due to how rails’ MySQL adapter handles this
+ # call ‘ActiveRecord::Base.connection.active?’, you’ll need
+ # to wrap the code that checks for a connection in in a mutex.
+ #
+ # ....I noticed this while working with a multi-core machine that
+ # was spawning multiple workling threads. Some of my workling
+ # threads would hit serious issues at this block of code without
+ # the mutex.
+ #
+ @mutex.synchronize do
+ unless ActiveRecord::Base.connection.active? # Keep MySQL connection alive
+ unless ActiveRecord::Base.connection.reconnect!
+ logger.fatal("Failed - Database not available!")
+ break
+ end
+ end
+ end
+
+ # Dispatch and process the messages
+ n = dispatch!(connection, clazz)
+ logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
+ sleep(self.class.sleep_time) unless n > 0
+
+ # If there is a memcache error, hang for a bit to give it a chance to fire up again
+ # and reset the connection.
+ rescue Workling::WorklingConnectionError
+ logger.warn("Listener thread #{clazz.name} failed to connect. Resetting connection.")
+ sleep(self.class.reset_time)
+ connection.reset
+ end
+ end
+
+ logger.debug("Listener thread #{clazz.name} ended")
+ end
+
+ # Dispatcher for one worker class. Will throw MemCacheError if unable to connect.
+ # Returns the number of worker methods called
+ def dispatch!(connection, clazz)
+ n = 0
+ for queue in @routing.queue_names_routing_class(clazz)
+ begin
+ result = connection.retrieve(queue)
+ if result
+ n += 1
+ handler = @routing[queue]
+ method_name = @routing.method_name(queue)
+ logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
+ handler.dispatch_to_worker_method(method_name, result)
+ end
+ rescue MemCache::MemCacheError => e
+ logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
+ raise e
+ end
+ end
+
+ return n
+ end
+ end
+ end
+ end
+end
View
35 vendor/plugins/workling/lib/workling/remote/runners/backgroundjob_runner.rb
@@ -0,0 +1,35 @@
+require 'workling/remote/runners/base'
+
+#
+# Use Ara Howards BackgroundJob to run the work. BackgroundJob loads Rails once per requested Job.
+# It persists over the database, and there is no requirement for separate processes to be started.
+# Since rails has to load before each request, it takes a moment for the job to run.
+#
+module Workling
+ module Remote
+ module Runners
+ class BackgroundjobRunner < Workling::Remote::Runners::Base
+ cattr_accessor :routing
+
+ def initialize
+ BackgroundjobRunner.routing =
+ Workling::Routing::ClassAndMethodRouting.new
+ end
+
+ # passes the job to bj by serializing the options to xml and passing them to
+ # ./script/bj_invoker.rb, which in turn routes the deserialized args to the
+ # appropriate worker.
+ def run(clazz, method, options = {})
+ stdin = @@routing.queue_for(clazz, method) +
+ " " +
+ options.to_xml(:indent => 0, :skip_instruct => true)
+
+ Bj.submit "./script/runner ./script/bj_invoker.rb",
+ :stdin => stdin
+
+ return nil # that means nothing!
+ end
+ end
+ end
+ end
+end
View
42 vendor/plugins/workling/lib/workling/remote/runners/base.rb
@@ -0,0 +1,42 @@
+#
+# Base class for Workling Runners.
+#
+# Runners must subclass this and implement the method
+#
+# Workling::Remote::Runners::Base#run(clazz, method, options = {})
+#
+# which is responsible for pushing the requested job into the background. Depending
+# on the Runner, this may require other code to dequeue the job. The actual
+# invocation of the runner should be done like this:
+#
+# Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+#
+# This ensures for consistent logging and handling of propagated exceptions. You can
+# also call the convenience method
+#
+# Workling::Remote::Runners::Base#dispatch!(clazz, method, options)
+#
+# which invokes this for you.
+#
+module Workling
+ module Remote
+ module Runners
+ class Base
+
+ # runner uses this to connect to a job broker
+ cattr_accessor :client
+
+ # default logger defined in Workling::Base.logger
+ def logger
+ Workling::Base.logger
+ end
+
+ # find the worker instance and invoke it. Invoking the worker method like this ensures for
+ # consistent logging and handling of propagated exceptions.
+ def dispatch!(clazz, method, options)
+ Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+ end
+ end
+ end
+ end
+end
View
45 vendor/plugins/workling/lib/workling/remote/runners/client_runner.rb
@@ -0,0 +1,45 @@
+require 'workling/remote/runners/base'
+require 'workling/clients/memcache_queue_client'
+
+#
+# Runs Jobs over a Client. The client should be a subclass of Workling::Client::Base.
+# Set the client like this:
+#
+# Workling::Remote::Runners::ClientRunner.client = Workling::Clients::AmqpClient.new
+#
+# Jobs are dispatched by requesting them on the Client. The Runner takes care of mapping of queue names to worker code.
+# this is done with Workling::ClassAndMethodRouting, but you can use your own by sublassing Workling::Routing.
+# Don’t worry about any of this if you’re not dealing directly with the queues.
+#
+# There’s a workling-client daemon that uses the configured invoker to retrieve work and dispatching these to the
+# responsible workers. If you intend to run this on a remote machine, then just check out your rails project
+# there and start up the workling client like this: ruby script/workling_client run.
+#
+module Workling
+ module Remote
+ module Runners
+ class ClientRunner < Workling::Remote::Runners::Base
+
+ # Routing class. Workling::Routing::ClassAndMethodRouting.new by default.
+ cattr_accessor :routing
+ @@routing ||= Workling::Routing::ClassAndMethodRouting.new
+
+ # The workling Client class. Workling::Clients::MemcacheQueueClient.new by default.
+ cattr_accessor :client
+ @@client ||= Workling::Clients::MemcacheQueueClient.new
+
+ # enqueues the job onto the client
+ def run(clazz, method, options = {})
+
+ # neet to connect in here as opposed to the constructor, since the EM loop is
+ # not available there.
+ @connected ||= self.class.client.connect
+
+ self.class.client.request(@@routing.queue_for(clazz, method), options)
+
+ return nil
+ end
+ end
+ end
+ end
+end
View
23 vendor/plugins/workling/lib/workling/remote/runners/not_remote_runner.rb
@@ -0,0 +1,23 @@
+require 'workling/remote/runners/base'
+
+#
+# directly dispatches to the worker method, in-process. options are first marshalled then dumped
+# in order to simulate the sideeffects of a remote call.
+#
+module Workling
+ module Remote
+ module Runners
+ class NotRemoteRunner < Workling::Remote::Runners::Base
+
+ # directly dispatches to the worker method, in-process. options are first marshalled then dumped
+ # in order to simulate the sideeffects of a remote call.
+ def run(clazz, method, options = {})
+ options = Marshal.load(Marshal.dump(options)) # get this to behave more like the remote runners
+ dispatch!(clazz, method, options)
+
+ return nil # nada. niente.
+ end
+ end
+ end
+ end
+end
View
38 vendor/plugins/workling/lib/workling/remote/runners/spawn_runner.rb
@@ -0,0 +1,38 @@
+require 'workling/remote/runners/base'
+
+#
+# Run the job over the spawn plugin. Refer to the README for instructions on
+# installing Spawn.
+#
+# Spawn forks the entire process once for each job. This means that the job starts
+# with a very low latency, but takes up more memory for each job.
+#
+# It's also possible to configure Spawn to start a Thread for each job. Do this
+# by setting
+#
+# Workling::Remote::Runners::SpawnRunner.options = { :method => :thread }
+#
+# Have a look at the Spawn README to find out more about the characteristics of this.
+#
+module Workling
+ module Remote
+ module Runners
+ class SpawnRunner < Workling::Remote::Runners::Base
+ cattr_accessor :options
+
+ # use thread for development and test modes. easier to hunt down exceptions that way.
+ @@options = { :method => (RAILS_ENV == "test" || RAILS_ENV == "development" ? :thread : :fork) }
+ include Spawn if Workling.spawn_installed?
+
+ # dispatches to Spawn, using the :fork option.
+ def run(clazz, method, options = {})
+ spawn(SpawnRunner.options) do # exceptions are trapped in here.
+ dispatch!(clazz, method, options)
+ end
+
+ return nil # that means nothing!
+ end
+ end
+ end
+ end
+end
View
13 vendor/plugins/workling/lib/workling/remote/runners/starling_runner.rb
@@ -0,0 +1,13 @@
+require 'workling/remote/runners/client_runner'
+
+#
+# DEPRECATED. Should use ClientRunner instead.
+#
+module Workling
+ module Remote
+ module Runners
+ class StarlingRunner < Workling::Remote::Runners::ClientRunner
+ end
+ end
+ end
+end
View
37 vendor/plugins/workling/lib/workling/return/store/base.rb
@@ -0,0 +1,37 @@
+#
+# Basic interface for getting and setting Data which needs to be passed between Workers and
+# client code.
+#
+module Workling
+ module Return
+ module Store
+ mattr_accessor :instance
+
+ # set a value in the store with the given key. delegates to the returnstore.
+ def self.set(key, value)
+ self.instance.set(key, value)
+ end
+
+ # get a value from the store. this should be destructive. delegates to the returnstore.
+ def self.get(key)
+ self.instance.get(key)
+ end
+
+ #
+ # Base Class for Return Stores. Subclasses need to implement set and get.
+ #
+ class Base
+
+ # set a value in the store with the given key.
+ def set(key, value)
+ raise NotImplementedError.new("set(key, value) not implemented in #{ self.class }")
+ end
+
+ # get a value from the store. this should be destructive.
+ def get(key)
+ raise NotImplementedError.new("get(key) not implemented in #{ self.class }")
+ end
+ end
+ end
+ end
+end
View
26 vendor/plugins/workling/lib/workling/return/store/memory_return_store.rb
@@ -0,0 +1,26 @@
+require 'workling/return/store/base'
+
+#
+# Stores directly into memory. This is for tests only - not for production use. aight?
+#
+module Workling
+ module Return
+ module Store
+ class MemoryReturnStore < Base
+ attr_accessor :sky
+
+ def initialize
+ self.sky = {}
+ end
+
+ def set(key, value)
+ self.sky[key] = value
+ end
+
+ def get(key)
+ self.sky.delete(key)
+ end
+ end
+ end
+ end
+end
View
31 vendor/plugins/workling/lib/workling/return/store/starling_return_store.rb
@@ -0,0 +1,31 @@
+require 'workling/return/store/base'