M/R phases working

app/bucket-mr.html.template

@@ -1,5 +1,6 @@
 <div id="mr_jobs">
 <form action="#/mapred/<%= bucket %>" method="post">
+  <input type="hidden" id="mr_phases" name="phases" value=""/>
   <select name="phase" multiple="multiple" title="Select map/reduce phases">
   </select>
   <button type="submit">Run</button>

app/key-mr.html.template

@@ -0,0 +1,3 @@
+<tr> 
+  <td colspan="7"><%= value %></td>
+</tr> 

app/rekon.js

@@ -1,3 +1,51 @@
+// Create a closure that acts as a continuation for a sequence of jQuery get
+// calls
+function phaseContinuation(count, options, finalizer) {
+  // Holds a list of the phase specs
+  phases = [ ];
+  return function(job_name, phase_data) {
+    console.log("Adding phase "+job_name);
+    count--;
+    json_ready = phase_data
+      .replace(/[\n\t\r]/g,' ')
+    phases[phases.length] = {name:job_name, spec:jQuery.parseJSON(json_ready)};
+    if (count == 0) { finalizer(phases, options); }
+  }
+}
+
+// This is the finalizer function of the continuation
+function runPhases(phases, options) {
+  bucket = options['bucket']
+  context = options['context']
+  var mapper = new RiakMapper(Rekon.client, bucket);
+  // p is an object {name:'job_name.map', spec:<json>}
+  phases.map(function(p) { addJob(mapper,p) });
+  mapper.run(null, function(status, list, xmlrequest) {
+    if (! status) {
+      context.render('bucket-err.html.template')
+        .replace('#keys tbody')
+        .wait();
+      return;
+    }
+
+    keyRows = list.map(function(obj) {
+      s = "";
+      // TODO Make formatting a custom function
+      for (k in obj) { s += k + " : " + obj[k] + "\n"; }
+      return {value:s}
+    });
+    context.renderEach('key-mr.html.template', keyRows)
+      .replace('#keys tbody')
+      .then(function(){ searchable('#bucket table tbody tr'); })
+  });
+}
+
+function addJob(mapper, p) {
+  if (/\.map$/i.test(p.name)) { mapper.map(p.spec); }
+  else if (/\.red$/i.test(p.name)) { mapper.reduce(p.spec); }
+  else { console.log("Skipping unknown file extension for "+p.name); }
+}
+
 rekonApp = Sammy(function() {
 
   $container = $(this.$element);
@@ -68,10 +116,11 @@ rekonApp = Sammy(function() {
       .wait();
     renderPhase('#mr_jobs select[multiple]', find_mrs, function() {
       $("#mr_jobs select[multiple]")
-        .asmSelect({sortable: true, animate: true, addItemTarget: 'top'});
+        .asmSelect({sortable: true, animate: true});
     });
-    context.render('bucket.html.template', {bucket: name})
-      .appendTo('#main');
+    context.render('bucket.html.template', {bucket:name})
+      .appendTo('#main')
+      .wait();
 
     bucket.keys(function(keys) {
       if (keys.length > 0) {
@@ -408,50 +457,23 @@ rekonApp = Sammy(function() {
       .attr('target', '_blank')
       .text('Riak').addClass('action'));
 
+    context.render('bucket-mr.html.template', {bucket:name})
+      .appendTo('#main')
+      .wait();
+    renderPhase('#mr_jobs select[multiple]', find_mrs, function() {
+      $("#mr_jobs select[multiple]")
+        .asmSelect({sortable: true, animate: true});
+    });
     context.render('bucket.html.template', {bucket:name})
       .appendTo('#main')
       .wait();
 
-    // TODO Assemble all phases together into a single job request
-    phase_url = Rekon.riakUrl('rekon.jobs/'+this.params['phase']);
-    jQuery.get(phase_url, function(data) {
-      phase_data = jQuery.parseJSON(data);
-      var mapper = new RiakMapper(Rekon.client, name);
-      mapper.map(phase_data);
-      mapper.run(null, function(status, list, xmlrequest) {
-        if (! status) {
-          context.render('bucket-err.html.template')
-            .replace('#keys tbody')
-            .wait();
-          return;
-        }
-
-        keyRows = list.map(
-          function(val) { return {bucket:name, key:val}; }
-        );
-        context.renderEach('key-row.html.template', keyRows)
-          .replace('#keys tbody')
-          .then(function(){ searchable('#bucket table tbody tr'); })
-          .then(function(){ renderPhase('#keys select#phase'); });
-
-        /*
-        switch(object.contentType) {
-        case 'image/png':
-        case 'image/jpeg':
-        case 'image/jpg':
-        case 'image/gif':
-          context.render('value-image.html.template', {bucket: name, key: key}).appendTo('#value');
-          return;
-        case 'application/json':
-          value = JSON.stringify(object, null, 4);
-          break;
-        default:
-          value = object;
-          break;
-        }
-        context.render('value-pre.html.template', {value: value}).appendTo('#value');
-        */
-      });
+    raw_phases = jQuery.parseJSON(this.params['phases']);
+    options = {bucket:name,context:context};
+    continuation = phaseContinuation(raw_phases.length, options, runPhases);
+    raw_phases.map(function(p) {
+      phase_url = Rekon.riakUrl('rekon.jobs/'+p);
+      jQuery.get(phase_url, function(data) { continuation(p, data) });
     });
   });
 
@@ -527,6 +549,16 @@ Rekon = {
 
 };
 
+// Set the form field to contain all selected elements
+$('#mr_jobs button').live('click', function(e){
+  var link = this;
+  //e.preventDefault();
+  var jobs = [ ];
+  $('#mr_jobs :selected')
+    .each(function(i, selected) { if (i>0) {jobs[i-1] = $(selected).val();} });
+  $('#mr_phases').val(JSON.stringify(jobs));
+});
+
 $('#keys a.move').live('click', function(e){
   var link = this;
   e.preventDefault();

buckets/alice/p1

@@ -0,0 +1,5 @@
+Alice was beginning to get very tired of sitting by her sister on the
+bank, and of having nothing to do: once or twice she had peeped into the
+book her sister was reading, but it had no pictures or conversations in
+it, 'and what is the use of a book,' thought Alice 'without pictures or
+conversation?'

buckets/alice/p2

@@ -0,0 +1,5 @@
+So she was considering in her own mind (as well as she could, for the
+hot day made her feel very sleepy and stupid), whether the pleasure
+of making a daisy-chain would be worth the trouble of getting up and
+picking the daisies, when suddenly a White Rabbit with pink eyes ran
+close by her.

buckets/alice/p5

@@ -0,0 +1,4 @@
+The rabbit-hole went straight on like a tunnel for some way, and then
+dipped suddenly down, so suddenly that Alice had not a moment to think
+about stopping herself before she found herself falling down a very deep
+well.

erlang/myjson.erl

@@ -1,11 +1,12 @@
 -module(myjson).
--export([encode/3]).
+-export([encode/3, encode/1]).
 
 encode(RiakObject, _, _) ->
   io:format("~p", [RiakObject]),
   encode(riak_object:get_value(RiakObject)).
 
 encode(<<"{}">>) -> [];
+encode({M,S,N}) -> [M,S,N];
 encode(Other) ->
   Term = binary_to_term(Other),
   [tl(tuple_to_list(Term))].

install.sh

@@ -1,58 +1,108 @@
-#!/bin/sh
-
-# Allow running the script and specifiying an install target
-#   ./install node-address:host
-if [ -n "$1" ] ; then
-  node=$1
-else
-  node="127.0.0.1:8098"
-fi
-
-riak_url="http://$node/riak/rekon"
-
-echo "Installing rekon to $node..."
-
-base_dir="`dirname $0`/app"
-
-# loop through everything in the app directory and put in in the rekon bucket
-for f in $(ls $base_dir); do
-  # echo "Uploading $f to riak"
-  case $f in
-    go | *.html )
-      content_type="text/html"
-      ;;
-    *.js )
-      content_type="application/javascript"
-      ;;
-    *.css )
-      content_type="text/css"
-      ;;
-    *.png )
-      content_type="image/png"
-      ;;
-    *.gif )
-      content_type="image/gif"
-      ;;
-    *.template )
-      content_type="application/x-sammy-template"
-      ;;
+#!/bin/bash
+# Author: Brian Lee Yung Rowe
+# Date: 2001.07.14
+# Rewritten from the original version by Adam Hunter
+
+do_exit()
+{
+  echo $1
+  exit $2
+}
+
+do_usage()
+{
+  do_exit "Usage: $0 [aejd] [host:port]" $1
+}
+
+content_type()
+{
+  case $1 in
+    go | *.html) content_type="text/html";;
+    *.js) content_type="application/javascript";;
+    *.css) content_type="text/css";;
+    *.png) content_type="image/png";;
+    *.gif) content_type="image/gif";;
+    *.template) content_type="application/x-sammy-template";;
+    *) content_type="text/plain";;
   esac
+  echo $content_type
+}
 
-  #echo "Uploading $f"
-  curl -X PUT -H"Content-Type: $content_type" $riak_url/$f --data-binary @$base_dir/$f
-done
+do_install_rekon()
+{
+  echo "Installing rekon to $node..."
+  riak_url="http://$node/riak/rekon"
+  base_dir="`dirname $0`/app"
+
+  for f in $(ls $base_dir); do
+    [ -n "$verbose" ] && echo "Uploading $f to rekon"
+    content_type=$(content_type $f)
+    curl -X PUT -H"Content-Type: $content_type" $riak_url/$f --data-binary @$base_dir/$f
+  done
+}
+
+do_install_jobs()
+{
+  echo "Installing jobs"
+  riak_url="http://$node/riak/rekon.jobs"
+  base_dir="`dirname $0`/jobs"
+  for f in $(ls $base_dir)
+  do
+    [ -n "$verbose" ] && echo "Uploading job $f to rekon.jobs"
+    content_type="application/javascript"
+    curl -X PUT -H"Content-Type: $content_type" $riak_url/$f --data-binary @$base_dir/$f
+  done
+}
+
+do_install_data()
+{
+  echo "Installing data"
+  riak_url="http://$node/riak"
+  base_dir=$(dirname $0)/buckets
+  for bucket in $(ls $base_dir)
+  do
+    dir=$base_dir/$bucket
+    [ -n "$verbose" ] && echo "Adding bucket $bucket"
+    for f in $(ls $dir)
+    do
+      [ -n "$verbose" ] && echo "Adding file as $bucket/$f"
+      content_type=$(content_type $f)
+      curl -X PUT -H"Content-Type: $content_type" $riak_url/$bucket/$f --data-binary @$dir/$f
+    done
+  done
+}
 
-riak_url="http://$node/riak/rekon.jobs"
-base_dir="`dirname $0`/jobs"
-for f in $(ls $base_dir)
+do_install_erlang()
+{
+  echo "Installing erlang modules to $module_dir"
+  sudo mkdir -p $module_dir
+  sudo erlc -o $module_dir erlang/*.erl
+
+}
+
+node="127.0.0.1:8098"
+module_dir=/etc/riak/erlang
+while getopts "aeE:jdv?" option
 do
-  content_type="application/javascript"
-  #echo "Uploading $f"
-  curl -X PUT -H"Content-Type: $content_type" $riak_url/$f --data-binary @$base_dir/$f
+  case $option in
+    a) erlang=yes; jobs=yes; data=yes;;
+    e) erlang=yes;;
+    E) erlang=yes; module_dir=$OPTARG;;
+    j) jobs=yes;;
+    d) data=yes;;
+    v) verbose=yes;;
+    '?') usage 0;;
+    *) usage 1;;
+  esac
 done
+shift $(($OPTIND - 1))
+
+[ -n "$1" ] && node=$1
+
+do_install_rekon
+[ -n "$jobs" ] && do_install_jobs
+[ -n "$data" ] && do_install_data
+[ -n "$erlang" ] && do_install_erlang
 
-module_dir=/etc/riak/erlang
-sudo mkdir -p $module_dir
-sudo erlc -o $module_dir erlang/*.erl
 
 echo "Installed, now visit: $riak_url/go"

jobs/count_words.map

@@ -0,0 +1,14 @@
+{"language":"javascript","source":"
+function(v) {
+  var m = v.values[0].data.toLowerCase().split(' ');
+  var r = [];
+  for(var i in m) {
+    if(m[i] != '') {
+      var o = {};
+      o[m[i]] = 1;
+      r.push(o);
+    }
+  }
+  return r;
+}
+"}

jobs/list_to_table.map

@@ -0,0 +1,6 @@
+{"language":"javascript", "source":"
+function(v,k,a) {
+  print(v);
+  return [ x ]; 
+}
+"}

jobs/sum_words.red

@@ -0,0 +1,12 @@
+{"language":"javascript","source":"
+function(v) {
+  var r = {};
+  for(var i in v) {
+    for(var w in v[i]) {
+      if(w in r) r[w] += v[i][w]; 
+      else r[w] = v[i][w];
+    }
+  }
+  return [r];
+}
+"}