initial import

igrigorik · Mar 3, 2009 · 64a0692 · 64a0692
commit 64a0692
Show file tree

Hide file tree

Showing 11 changed files with 118 additions and 0 deletions.
diff --git a/README.rdoc b/README.rdoc
@@ -0,0 +1,25 @@
+= Browser / HTTP Map-Reduce driver
+
+Working example of browser based, HTTP driven map-reduce for a distributed
+word count. The jobs are assigned by the job-server and computation is done
+by the clients browser, after which the results are posted back (emit) to the
+server and the cycle repeats until the computation is done.
+
+== Huh, why?
+
+It's a proof of concept with the goal of lowering the barrier to entry for any online
+user to contribute some of their spare CPU cycles to a distributed computation.
+After all, what could be easier than clicking a link?
+
+== Launching example
+
+  (1) > ruby jobserver.rb -p 80 -e production
+  (2) > open your browser and point it to the jobserver
+
+  That's it.
+
+For full details, take a look at:
+http://www.igvita.com/2009/03/03/collaborative-map-reduce-in-the-browser/
+
+== Contact
+Ilya Grigorik <ilya@igvita.com> (Twitter: @igrigorik)
diff --git a/data/1.txt b/data/1.txt
@@ -0,0 +1 @@
+1 2 3 4 5
diff --git a/data/2.txt b/data/2.txt
@@ -0,0 +1 @@
+4 5 6 7 8 9 10
diff --git a/data/3.txt b/data/3.txt
@@ -0,0 +1 @@
+11 12 13 14 15
diff --git a/data/4.txt b/data/4.txt
@@ -0,0 +1 @@
+16 17 18 19 20
diff --git a/data/5.txt b/data/5.txt
@@ -0,0 +1 @@
+21 22 23 24 25
diff --git a/jobserver.rb b/jobserver.rb
@@ -0,0 +1,40 @@
+# Ilya Grigorik - February 28 / '09
+#
+# HTTP driver for a browser based Map-Reduce
+#
+#  1) Server redirects the client to available jobs
+#  2) Client executes javascript map/reduce jobs via JavaScript
+#  3) Client emits (POST) intermediate results back to job server
+#  4) Client is redirected to next available job (map or reduce)
+#  
+
+require "rubygems"
+require "sinatra"
+
+configure do
+  set :map_jobs, Dir.glob("data/*.txt")
+  set :reduce_jobs, []
+  set :result, nil
+end
+
+get "/" do
+  redirect "/map/#{options.map_jobs.pop}" unless options.map_jobs.empty?
+  redirect "/reduce"                      unless options.reduce_jobs.empty?
+  redirect "/done"
+end
+
+get "/map/*"  do erb :map,    :file => params[:splat].first; end
+get "/reduce" do erb :reduce, :data => options.reduce_jobs;  end
+get "/done"   do erb :done,   :answer => options.result;     end
+
+post "/emit/:phase" do
+  case params[:phase]
+  when "reduce" then
+    options.reduce_jobs.push params['count']
+    redirect "/"
+
+  when "finalize" then
+    options.result = params['sum']
+    redirect "/done"
+  end
+end
diff --git a/views/done.erb b/views/done.erb
@@ -0,0 +1,4 @@
+<%= options[:answer] %>
+
+<!-- Another client could be doing the final reduce, poll to see result -->
+<meta http-equiv="refresh" content="2" />
diff --git a/views/layout.erb b/views/layout.erb
@@ -0,0 +1,40 @@
+<html>
+  <head>
+    <script type="text/javascript">
+
+      function map() {
+        var words = document.body.innerHTML.split(/\n|\s/).length;
+        emit('reduce', {'count': words});
+      }
+
+      function reduce() {
+        var sum = 0;
+        var docs = document.body.innerHTML.split(/\n/);
+        for each (num in docs) { sum+= parseInt(num) > 0 ? parseInt(num) : 0 }
+        emit('finalize', {'sum': sum});
+      }
+
+      function emit(phase, data) {
+        var myForm = document.createElement("form");
+        myForm.method="POST";
+        myForm.action = "/emit/"+phase;
+
+        for (var k in data) {
+          var myInput = document.createElement("input") ;
+          myInput.setAttribute("name", k) ;
+          myInput.setAttribute("value", data[k]);
+          myForm.appendChild(myInput) ;
+        }
+
+        document.body.appendChild(myForm) ;
+        myForm.submit() ;
+        document.body.removeChild(myForm) ;
+      }
+
+    </script>
+  </head>
+
+  <body onload="<%= "#{@execute}();" if @execute %>">
+    <%= yield %>
+  </body>
+</html>
diff --git a/views/map.erb b/views/map.erb
@@ -0,0 +1,2 @@
+<% @execute = "map" %>
+<%= File.open(options[:file], "r").readlines.join %>
diff --git a/views/reduce.erb b/views/reduce.erb
@@ -0,0 +1,2 @@
+<% @execute = "reduce" %>
+<%= options[:data].join("\n") %>