Allows An Experiment To Have Multiple Goals

README.mdown

@@ -314,12 +314,34 @@ Your code may then track a completion using the metric instead of
 the experiment name:
 
     finished(:conversion)
-    
-You can also create a new metric by instantiating and saving a new Metric object. 
+
+You can also create a new metric by instantiating and saving a new Metric object.
 
     Split::Metric.new(:conversion)
     Split::Metric.save
 
+#### Goals
+
+You might wish to allow an experiment to have multiple goals.
+The API to define goals for an experiment is this:
+
+    ab_test({"link_color" => ["purchase", "refund"]}, "red", "blue")
+
+or you can you can define them in a configuration file:
+
+    Split.configure do |config|
+      config.experiments = {
+        :link_color => {
+          :alternatives => ["red", "blue"],
+          :goals => ["purchase", "refund"]
+        }
+      }
+    end
+
+To complete a goal conversion, you do it like:
+
+    finished({"link_color" => ["purchase"]})
+
 ### DB failover solution
 
 Due to the fact that Redis has no autom. failover mechanism, it's
@@ -388,7 +410,7 @@ is configured.
 
 ## Outside of a Web Session
 
-Split provides the Helper module to facilitate running experiments inside web sessions. 
+Split provides the Helper module to facilitate running experiments inside web sessions.
 
 Alternatively, you can access the underlying Metric, Trial, Experiment and Alternative objects to
 conduct experiments that are not tied to a web session.
@@ -412,11 +434,11 @@ end
 
 ## Algorithms
 
-By default, Split ships with an algorithm that randomly selects from possible alternatives for a traditional a/b test. 
+By default, Split ships with an algorithm that randomly selects from possible alternatives for a traditional a/b test.
 
-An implementation of a bandit algorithm is also provided. 
+An implementation of a bandit algorithm is also provided.
 
-Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file. 
+Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file.
 
 ## Extensions
 

lib/split/alternative.rb

@@ -19,50 +19,71 @@ def to_s
       name
     end
 
+    def goals
+      self.experiment.goals
+    end
+
     def participant_count
-      @participant_count ||= Split.redis.hget(key, 'participant_count').to_i
+      Split.redis.hget(key, 'participant_count').to_i
     end
 
     def participant_count=(count)
-      @participant_count = count
       Split.redis.hset(key, 'participant_count', count.to_i)
     end
 
-    def completed_count
-      @completed_count ||= Split.redis.hget(key, 'completed_count').to_i
+    def completed_count(goal = nil)
+      field = set_field(goal)
+      Split.redis.hget(key, field).to_i
+    end
+
+    def all_completed_count
+      if goals.empty?
+        completed_count
+      else
+        goals.inject(completed_count) do |sum, g|
+          sum + completed_count(g)
+        end
+      end
     end
-    
+
     def unfinished_count
-      participant_count - completed_count
+      participant_count - all_completed_count
+    end
+
+    def set_field(goal)
+      field = "completed_count"
+      field += ":" + goal unless goal.nil?
+      return field
     end
 
-    def completed_count=(count)
-      @completed_count = count
-      Split.redis.hset(key, 'completed_count', count.to_i)
+    def set_completed_count (count, goal = nil)
+      field = set_field(goal)
+      Split.redis.hset(key, field, count.to_i)
     end
 
     def increment_participation
-      @participant_count = Split.redis.hincrby key, 'participant_count', 1
+      Split.redis.hincrby key, 'participant_count', 1
     end
 
-    def increment_completion
-       @completed_count = Split.redis.hincrby key, 'completed_count', 1
+    def increment_completion(goal = nil)
+      field = set_field(goal)
+      Split.redis.hincrby(key, field, 1)
     end
 
     def control?
       experiment.control.name == self.name
     end
 
-    def conversion_rate
+    def conversion_rate(goal = nil)
       return 0 if participant_count.zero?
-      (completed_count.to_f/participant_count.to_f)
+      (completed_count(goal).to_f)/participant_count.to_f
     end
 
     def experiment
       Split::Experiment.find(experiment_name)
     end
 
-    def z_score
+    def z_score(goal = nil)
       # CTR_E = the CTR within the experiment split
       # CTR_C = the CTR within the control split
       # E = the number of impressions within the experiment split
@@ -74,8 +95,9 @@ def z_score
 
       return 'N/A' if control.name == alternative.name
 
-      ctr_e = alternative.conversion_rate
-      ctr_c = control.conversion_rate
+      ctr_e = alternative.conversion_rate(goal)
+      ctr_c = control.conversion_rate(goal)
+
 
       e = alternative.participant_count
       c = control.participant_count
@@ -93,9 +115,13 @@ def save
     end
 
     def reset
-      @participant_count = nil
-      @completed_count = nil
       Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0
+      unless goals.empty?
+        goals.each do |g|
+          field = "completed_count:#{g}"
+          Split.redis.hset key, field, 0
+        end
+      end
     end
 
     def delete

lib/split/configuration.rb

@@ -48,7 +48,7 @@ def metrics
       end
       @metrics
     end
-    
+
     def normalized_experiments
       if @experiments.nil?
         nil
@@ -58,13 +58,13 @@ def normalized_experiments
           experiment_config[name] = {}
         end
         @experiments.each do | experiment_name, settings|
-          experiment_config[experiment_name][:alternatives] = normalize_alternatives(settings[:alternatives]) if settings[:alternatives] 
+          experiment_config[experiment_name][:alternatives] = normalize_alternatives(settings[:alternatives]) if settings[:alternatives]
+          experiment_config[experiment_name][:goals] = settings[:goals] if settings[:goals]
         end
         experiment_config
       end
     end
-
-
+
     def normalize_alternatives(alternatives)
       given_probability, num_with_probability = alternatives.inject([0,0]) do |a,v|
         p, n = a

lib/split/dashboard/public/style.css

@@ -160,6 +160,10 @@ body {
   margin:30px 0;
 }
 
+.experiment_with_goal {
+  margin: -32px 0 30px 0;
+}
+
 .experiment .experiment-header {
   background: #f4f4f4;
   background: -webkit-gradient(linear, left top, left bottom,
@@ -177,14 +181,18 @@ body {
 
 .experiment h2 {
   color:#888;
-  margin: 12px 0 0;
+  margin: 12px 0 12px 0;
   font-size: 1em;
   font-weight:bold;
   float:left;
   text-shadow:0 1px 0 rgba(255,255,255,0.8);
 }
 
-.experiment h2 .version{
+.experiment h2 .goal {
+  font-style: italic;
+}
+
+.experiment h2 .version {
   font-style:italic;
   font-size:0.8em;
   color:#bbb;

lib/split/dashboard/views/_experiment.erb

@@ -1,20 +1,28 @@
-<div class="experiment">
+<% unless goal.nil? %>
+  <% experiment_class = "experiment experiment_with_goal" %>
+<% else %>
+  <% experiment_class = "experiment" %>
+<% end %>
+<div class="<%= experiment_class %>">
   <div class="experiment-header">
     <h2>
       Experiment: <%= experiment.name %>
       <% if experiment.version > 1 %><span class='version'>v<%= experiment.version %></span><% end %>
+      <% unless goal.nil? %><span class='goal'>Goal:<%= goal %></span><% end %>
     </h2>
 
-    <div class='inline-controls'>
-      <small><%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %></small>
-      <form action="<%= url "/reset/#{experiment.name}" %>" method='post' onclick="return confirmReset()">
-        <input type="submit" value="Reset Data">
-      </form>
-      <form action="<%= url "/#{experiment.name}" %>" method='post' onclick="return confirmDelete()">
-        <input type="hidden" name="_method" value="delete"/>
-        <input type="submit" value="Delete" class="red">
-      </form>
-    </div>
+    <% if goal.nil? %>
+      <div class='inline-controls'>
+        <small><%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %></small>
+        <form action="<%= url "/reset/#{experiment.name}" %>" method='post' onclick="return confirmReset()">
+          <input type="submit" value="Reset Data">
+        </form>
+        <form action="<%= url "/#{experiment.name}" %>" method='post' onclick="return confirmDelete()">
+          <input type="hidden" name="_method" value="delete"/>
+          <input type="submit" value="Delete" class="red">
+        </form>
+      </div>
+    <% end %>
   </div>
   <table>
     <tr>
@@ -27,7 +35,7 @@
       <th>Finish</th>
     </tr>
 
-    <% total_participants = total_completed = 0 %>
+    <% total_participants = total_completed = total_unfinished = 0 %>
     <% experiment.alternatives.each do |alternative| %>
       <tr>
         <td>
@@ -38,23 +46,23 @@
         </td>
         <td><%= alternative.participant_count %></td>
         <td><%= alternative.unfinished_count %></td>
-        <td><%= alternative.completed_count %></td>
+        <td><%= alternative.completed_count(goal) %></td>
         <td>
-          <%= number_to_percentage(alternative.conversion_rate) %>%
-          <% if experiment.control.conversion_rate > 0 && !alternative.control? %>
-            <% if alternative.conversion_rate > experiment.control.conversion_rate %>
+          <%= number_to_percentage(alternative.conversion_rate(goal)) %>%
+          <% if experiment.control.conversion_rate(goal) > 0 && !alternative.control? %>
+            <% if alternative.conversion_rate(goal) > experiment.control.conversion_rate(goal) %>
               <span class='better'>
-                +<%= number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1) %>%
+                +<%= number_to_percentage((alternative.conversion_rate(goal)/experiment.control.conversion_rate(goal))-1) %>%
               </span>
-            <% elsif alternative.conversion_rate < experiment.control.conversion_rate %>
+            <% elsif alternative.conversion_rate(goal) < experiment.control.conversion_rate(goal) %>
               <span class='worse'>
-                <%= number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1) %>%
+                <%= number_to_percentage((alternative.conversion_rate(goal)/experiment.control.conversion_rate(goal))-1) %>%
               </span>
             <% end  %>
           <% end %>
         </td>
         <td>
-          <span title='z-score: <%= round(alternative.z_score, 3) %>'><%= confidence_level(alternative.z_score) %></span>
+          <span title='z-score: <%= round(alternative.z_score(goal), 3) %>'><%= confidence_level(alternative.z_score(goal)) %></span>
         </td>
         <td>
           <% if experiment.winner %>
@@ -73,13 +81,14 @@
       </tr>
 
       <% total_participants += alternative.participant_count %>
-      <% total_completed += alternative.completed_count %>
+      <% total_unfinished += alternative.unfinished_count %>
+      <% total_completed += alternative.completed_count(goal) %>
     <% end %>
 
     <tr class="totals">
       <td>Totals</td>
       <td><%= total_participants %></td>
-      <td><%= total_participants - total_completed %></td>
+      <td><%= total_unfinished %></td>
       <td><%= total_completed %></td>
       <td>N/A</td>
       <td>N/A</td>

lib/split/dashboard/views/_experiment_with_goal_header.erb

@@ -0,0 +1,14 @@
+<div class="experiment">
+  <div class="experiment-header">
+      <div class='inline-controls'>
+        <small><%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %></small>
+        <form action="<%= url "/reset/#{experiment.name}" %>" method='post' onclick="return confirmReset()">
+          <input type="submit" value="Reset Data">
+        </form>
+        <form action="<%= url "/#{experiment.name}" %>" method='post' onclick="return confirmDelete()">
+          <input type="hidden" name="_method" value="delete"/>
+          <input type="submit" value="Delete" class="red">
+        </form>
+      </div>
+  </div>
+</div>

lib/split/dashboard/views/index.erb

@@ -2,7 +2,14 @@
   <p class="intro">The list below contains all the registered experiments along with the number of test participants, completed and conversion rate currently in the system.</p>
 
   <% @experiments.each do |experiment| %>
-    <%= erb :_experiment, :locals => {:experiment => experiment} %>
+    <% if experiment.goals.empty? %>
+      <%= erb :_experiment, :locals => {:goal => nil, :experiment => experiment} %>
+    <% else %>
+      <%= erb :_experiment_with_goal_header, :locals => {:experiment => experiment} %>
+      <% experiment.goals.each do |g| %>
+        <%= erb :_experiment, :locals => {:goal => g, :experiment => experiment} %>
+      <% end %>
+    <% end %>
   <% end %>
 <% else %>
   <p class="intro">No experiments have started yet, you need to define them in your code and introduce them to your users.</p>