@@ -21,5 +21,6 @@ lib/pandemic/util.rb lib/pandemic.rb Manifest MIT-LICENSE +pandemic.gemspec Rakefile README.markdown Manifest @@ -1,7 +1,7 @@ # Pandemic -Pandemic is a map-reduce framework. You give it the map, process, and reduce methods and it handles the rest. It's designed to serve requests in real-time, but can also be used for offline tasks. +Pandemic is a map-reduce-ish framework. It allows you to partition requests and distribute them as you please, then process the request (or parts of it) on any number of nodes, and then reduce the response however you please. It's designed to serve requests in real-time, but can also be used for offline tasks. -It's different from the typical map-reduce framework in that it doesn't have a master-worker structure. Every node can map, process, and reduce. It also doesn't have the concept of jobs, everything is a request. +It's different from the typical map-reduce framework in that it doesn't have a master-worker structure. Every node does can do everything. It's actually not strictly a map-reduce framework, it's a bit more lenient on what you can do to your data/request other than map and reduce. The framework is designed to be as flexible as possible, there is no rigid request format, or API, you can specify it however you want. You can send it http-style headers and a body, you can send it JSON, or you can even just send it a single line and have it do whatever you want. The only requirement is that you write your handler to appropriately act on the request and return the response. @@ -21,9 +21,9 @@ The framework is designed to be as flexible as possible, there is no rigid reque pandemic_server.handler = Handler # Pandemic will call the initializer once per process pandemic_server.start.join -In this example, the handler doesn't define the map or reduce methods, and the defaults are used. The default for each is as follows: +In this example, the handler doesn't define the partition or reduce methods, and the defaults are used. The default for each is as follows: - * map: Send the full request body to every connected node + * partition: Send the full request body to every connected node * process: Return the body (do nothing) * reduce: Concatenate all the responses @@ -117,6 +117,8 @@ The servers are going to try to bind to localhost:4000 and localhost:4001 so mak By default, the handler runs in the same Ruby process as Pandemic. By setting the fork\_for\_processor to true in pandemic\_server.yml, you can have Pandemic fork to new processes to run the process method. This is particularly useful when your process method goes to MySQL which locks the entire process until MySQL returns. ## Change History +Version 0.3.1 + * Changed map to partition to more accurately reflect what it does. This breaks backwards compatibility, but all you have to do is rename your method. Version 0.3.0 * Pandemic can now fork to call the process method README.markdown @@ -3,7 +3,7 @@ require 'pandemic' require 'json' class WordCounter < Pandemic::ServerSide::Handler - def map(request, servers) + def partition(request, servers) # select only the alive servers (non-disconnected) only_alive = servers.keys.select{|k| servers[k] != :disconnected} examples/server/word_count_server.rb @@ -4,8 +4,8 @@ module Pandemic def config Config end - - def map(request, servers) + + def partition(request, servers) map = {} servers.each do |server, status| if status != :disconnected lib/pandemic/server_side/handler.rb @@ -115,7 +115,7 @@ module Pandemic def handle_client_request(request) info("Handling client request") - map = @handler_instance.map(request, connection_statuses) + map = @handler_instance.partition(request, connection_statuses) request.max_responses = map.size debug("Sending client request to #{map.size} handlers (#{request.hash})") lib/pandemic/server_side/server.rb @@ -6,23 +6,22 @@ Gem::Specification.new do |s| s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version= s.authors = ["Arya Asemanfar"] - s.date = %q{2009-05-27} + s.date = %q{2009-06-18} s.description = %q{Distribute MapReduce to any of the workers and it will spread, like a pandemic.} s.email = %q{aryaasemanfar@gmail.com} s.extra_rdoc_files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "README.markdown"] - s.files = ["examples/client/client.rb", "examples/client/constitution.txt", "examples/client/pandemic_client.yml", "examples/server/pandemic_server.yml", "examples/server/word_count_server.rb", "lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Manifest", "MIT-LICENSE", "Rakefile", "README.markdown", "pandemic.gemspec", "test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/processor_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"] - s.has_rdoc = true + s.files = ["examples/client/client.rb", "examples/client/constitution.txt", "examples/client/pandemic_client.yml", "examples/server/pandemic_server.yml", "examples/server/word_count_server.rb", "lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Manifest", "MIT-LICENSE", "pandemic.gemspec", "Rakefile", "README.markdown", "test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/processor_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"] s.homepage = %q{https://github.com/arya/pandemic/} s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Pandemic", "--main", "README.markdown"] s.require_paths = ["lib"] s.rubyforge_project = %q{pandemic} - s.rubygems_version = %q{1.3.1} + s.rubygems_version = %q{1.3.4} s.summary = %q{Distribute MapReduce to any of the workers and it will spread, like a pandemic.} s.test_files = ["test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/processor_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"] if s.respond_to? :specification_version then current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION - s.specification_version = 2 + s.specification_version = 3 if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then s.add_development_dependency(%q<shoulda>, [">= 0"]) pandemic.gemspec @@ -21,7 +21,7 @@ class HandlerTest < Test::Unit::TestCase should "map to all non-disconnected nodes" do @request.expects(:body).twice.returns("123") - map = @handler.map(@request, @servers) + map = @handler.partition(@request, @servers) # see setup for @servers assert_equal 2, map.size assert_equal "123", map[1] test/handler_test.rb @@ -94,7 +94,7 @@ class ServerTest < Test::Unit::TestCase request = mock() request.expects(:hash).at_least_once.returns("abcddef134123") @peer.expects(:connected?).returns(true) - handler.expects(:map).with(request, is_a(Hash)).returns({"localhost:4000" => "1", "localhost:4001" => "2"}) + handler.expects(:partition).with(request, is_a(Hash)).returns({"localhost:4000" => "1", "localhost:4001" => "2"}) request.expects(:max_responses=).with(2) @peer.expects(:client_request).with(request, "2") test/server_test.rb 52aa963b0ce0ac8d2c13a63c01b8915fa2d376ac Arya Asemanfar arya.asemanfar@gmail.com http://github.com/arya/pandemic/commit/b239dc3da32d7af06e3427decf8a67e9bb709810 b239dc3da32d7af06e3427decf8a67e9bb709810 2009-06-18T22:20:27-07:00 2009-06-18T22:20:27-07:00 renamed map to partition because i'm a noob e7fdec410376af4d84c4145bb7bf832af53e9167 Arya Asemanfar arya.asemanfar@gmail.com