@@ -1,3 +1,7 @@ +Version 0.4.0 +- refactored pollers. now now longer mainly about starling +- refactored starling client, converted to generalized memcachequeue client. + Version 0.3.1, 15.10.08 - fixed to autodiscovery code bugs. - introduced Workling::VERSION CHANGES.markdown @@ -75,7 +75,7 @@ You'll see that this executes pretty much instantly. Run 'top' in another termin You cannot run your workers on a remote machine or cluster them with spawn. You also have no persistence: if you've fired of a lot of work and everything dies, there's no way of picking up where you left off. -# Using the Starling runner instead of Spawn +# Using the Starling runner If you want cross machine jobs with low latency and a low memory overhead, you might want to look into using the Starling Runner. @@ -153,6 +153,21 @@ You might wonder what exactly starling does. Here's a little snippet you can pla 11 starling = MemCache.new('localhost:22122') 12 loop { puts starling.get('my_queue') } 13 + +# Using RudeQueue + +RudeQueue is a Starling-like Queue that runs on top of your database and requires no extra processes. Use this if you don't need very fast job processing and want to avoid managing the extra process starling requires. + +Install the RudeQ plugin like this: + + 1 ./script/plugin install git://github.com/matthewrudy/rudeq.git + 2 rake queue:setup + 3 rake db:migrate + +Configure Workling to use RudeQ. Add this to your environment: + + Workling::Clients::MemcacheQueue.memcache_client_class = RudeQ::Client + Workling::Remote.dispatcher = Workling::Remote::Runners::StarlingRunner.new # Using BackgroundJob README.markdown @@ -1,2 +1,3 @@ Workling.try_load_a_memcache_client -Workling::Discovery.discover! \ No newline at end of file +Workling::Discovery.discover! +Workling.config \ No newline at end of file init.rb @@ -20,7 +20,7 @@ module Workling mattr_accessor :load_path @@load_path = File.expand_path(File.join(File.dirname(__FILE__), '../../../../app/workers')) - VERSION = "0.3.1" + VERSION = "0.4.0" # # determine the runner to use if nothing is specifically set. workling will try to detect @@ -108,14 +108,15 @@ module Workling # returns a config hash. reads RAILS_ROOT/config/workling.yml # def self.config - config_path = File.join(RAILS_ROOT, 'config', 'workling.yml') - @@config ||= YAML.load_file(config_path)[RAILS_ENV || 'development'].symbolize_keys - - # make sure the config file could be read correctly. - raise ConfigurationError.new unless @@config - - @@config[:memcache_options].symbolize_keys! if @@config[:memcache_options] - @@config + begin + config_path = File.join(RAILS_ROOT, 'config', 'workling.yml') + @@config ||= YAML.load_file(config_path)[RAILS_ENV || 'development'].symbolize_keys + @@config[:memcache_options].symbolize_keys! if @@config[:memcache_options] + @@config + rescue + # config files could not be read correctly + raise ConfigurationError.new + end end private lib/workling.rb @@ -11,6 +11,10 @@ module Workling module Clients class MemcacheQueue + # the class with which the connection is instantiated + cattr_accessor :memcache_client_class + @@memcache_client_class ||= ::MemCache + # the url with which the memcache client expects to reach starling attr_accessor :queueserver_urls @@ -28,7 +32,7 @@ module Workling def initialize @queueserver_urls = Workling.config[:listens_on].split(',').map { |url| url ? url.strip : url } options = [@queueserver_urls, Workling.config[:memcache_options]].compact - @connection = ::MemCache.new(*options) + @connection = MemcacheQueue.memcache_client_class.new(*options) raise_unless_connected! end @@ -44,7 +48,7 @@ module Workling begin @connection.stats rescue - raise Workling::QeueuserverNotFoundError.new + raise Workling::QueueserverNotFoundError.new end end end lib/workling/clients/memcache_queue.rb @@ -12,7 +12,7 @@ module Workling # Seconds to wait while resetting connection cattr_accessor :reset_time - def initialize(routing, client_class = Workling::Clients::MemcacheQueue) + def initialize(routing, client_class) Poller.sleep_time = Workling.config[:sleep_time] || 2 Poller.reset_time = Workling.config[:reset_time] || 30 lib/workling/remote/invokers/poller.rb @@ -23,6 +23,9 @@ module Workling module Runners class Base + # runner uses this to connect to a job broker + cattr_accessor :client + # default logger defined in Workling::Base.logger def logger Workling::Base.logger lib/workling/remote/runners/base.rb @@ -17,7 +17,6 @@ module Workling module Runners class StarlingRunner < Workling::Remote::Runners::Base cattr_accessor :routing - cattr_accessor :client def initialize StarlingRunner.client = Workling::Clients::MemcacheQueue.new lib/workling/remote/runners/starling_runner.rb @@ -11,7 +11,8 @@ puts '** Use CTRL-C to stop.' ActiveRecord::Base.logger = Workling::Base.logger ActionController::Base.logger = Workling::Base.logger -poller = Workling::Starling::Poller.new(Workling::Routing::ClassAndMethodRouting.new) +client = Workling::Remote.dispatcher.client +poller = Workling::Starling::Poller.new(Workling::Routing::ClassAndMethodRouting.new, client) trap(:INT) { poller.stop; exit } script/listen.rb @@ -6,7 +6,7 @@ context "the dispatcher poller" do # the memoryreturnstore behaves exactly like memcache. @connection = Workling::Return::Store::MemoryReturnStore.new - @client = Workling::Remote::Invokers::Poller.new(routing) + @client = Workling::Remote::Invokers::Poller.new(routing, Workling::Clients::MemcacheQueue) end specify "should invoke Util.echo with the arg 'hello' if the string 'hello' is set onto the queue utils__echo" do test/dispacher_poller_test.rb 6a99f1b35ce16430a0368018e2a2ff28de395616 Rany Keddo rany@playtype.net http://github.com/digitalhobbit/workling/commit/6e9cee2d8f7c00ad6d8f5340b544beeab0cdc698 6e9cee2d8f7c00ad6d8f5340b544beeab0cdc698 2008-11-03T05:24:36-08:00 2008-11-03T05:24:36-08:00 more refactoring to accomodate rudeq 1ee050718a8fcf68e03ccf41c9572cab08c12b34 Rany Keddo rany@playtype.net