Pluggable Algorithms

README.mdown

@@ -264,7 +264,7 @@ in a hash or a configuration file:
       config.experiments = YAML.load_file "config/experiments.yml"
     end
 
-This hash can control your experiment's variants, weights, and if the
+This hash can control your experiment's variants, weights, algorithm and if the
 experiment resets once finished:
 
     Split.configure do |config|
@@ -274,6 +274,7 @@ experiment resets once finished:
           :resettable => false,
         },
         :my_second_experiment => {
+          :algorithm => 'Split::Algorithms::Whiplash',
           :variants => [
             { :name => "a", :percent => 67 },
             { :name => "b", :percent => 33 },
@@ -310,6 +311,11 @@ Your code may then track a completion using the metric instead of
 the experiment name:
 
     finished(:conversion)
+
+You can also create a new metric by instantiating and saving a new Metric object. 
+
+    Split::Metric.new(:conversion)
+    Split::Metric.save
 
 ### DB failover solution
 
@@ -377,6 +383,38 @@ Split.redis.namespace = "split:blog"
 We recommend sticking this in your initializer somewhere after Redis
 is configured.
 
+## Outside of a Web Session
+
+Split provides the Helper module to facilitate running experiments inside web sessions. 
+
+Alternatively, you can access the underlying Metric, Trial, Experiment and Alternative objects to
+conduct experiments that are not tied to a web session.
+
+```ruby
+# create a new experiment
+experiment = Split::Experiment.find_or_create('color', 'red', 'blue')
+# create a new trial
+trial = Trial.new(:experiment => experiment)
+# run trial
+trial.choose!
+# get the result, returns either red or blue
+trial.alternative.name
+
+# if the goal has been achieved, increment the successful completions for this alternative.
+if goal_acheived?
+  trial.complete!
+end
+
+```
+
+## Algorithms
+
+By default, Split ships with an algorithm that randomly selects from possible alternatives for a traditional a/b test. 
+
+An implementation of a bandit algorithm is also provided. 
+
+Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file. 
+
 ## Extensions
 
   - [Split::Export](http://github.com/andrew/split-export) - easily export ab test data out of Split

lib/split.rb

@@ -1,4 +1,4 @@
-%w[experiment alternative helper version configuration persistence exceptions].each do |f|
+%w[algorithms extensions metric trial experiment alternative helper version configuration persistence exceptions].each do |f|
   require "split/#{f}"
 end
 

lib/split/algorithms.rb

@@ -0,0 +1,3 @@
+%w[weighted_sample whiplash].each do |f|
+  require "split/algorithms/#{f}"
+end

lib/split/algorithms/weighted_sample.rb

@@ -0,0 +1,17 @@
+module Split
+  module Algorithms
+    module WeightedSample
+      def self.choose_alternative(experiment)
+        weights = experiment.alternatives.map(&:weight)
+
+        total = weights.inject(:+)
+        point = rand * total
+
+        experiment.alternatives.zip(weights).each do |n,w|
+          return n if w >= point
+          point -= w
+        end
+      end
+    end
+  end
+end

lib/split/algorithms/whiplash.rb

@@ -0,0 +1,35 @@
+# A multi-armed bandit implementation inspired by 
+# @aaronsw and victorykit/whiplash
+require 'simple-random'
+
+module Split
+  module Algorithms
+    module Whiplash
+      def self.choose_alternative(experiment)
+        experiment[best_guess(experiment.alternatives)]
+      end
+
+      private
+
+      def self.arm_guess(participants, completions)
+        a = [participants, 0].max
+        b = [participants-completions, 0].max
+        s = SimpleRandom.new; s.set_seed; s.beta(a+fairness_constant, b+fairness_constant)
+      end
+
+      def self.best_guess(alternatives)
+        guesses = {}
+        alternatives.each do |alternative| 
+          guesses[alternative.name] = arm_guess(alternative.participant_count, alternative.completed_count) 
+        end
+        gmax = guesses.values.max
+        best = guesses.keys.select {|name| guesses[name] ==  gmax }
+        return best.sample
+      end
+
+      def self.fairness_constant
+        7
+      end
+    end
+  end
+end

lib/split/alternative.rb

@@ -20,31 +20,33 @@ def to_s
     end
 
     def participant_count
-      Split.redis.hget(key, 'participant_count').to_i
+      @participant_count ||= Split.redis.hget(key, 'participant_count').to_i
     end
 
     def participant_count=(count)
+      @participant_count = count
       Split.redis.hset(key, 'participant_count', count.to_i)
     end
 
     def completed_count
-      Split.redis.hget(key, 'completed_count').to_i
+      @completed_count ||= Split.redis.hget(key, 'completed_count').to_i
     end
 
     def unfinished_count
       participant_count - completed_count
     end
 
     def completed_count=(count)
+      @completed_count = count
       Split.redis.hset(key, 'completed_count', count.to_i)
     end
 
     def increment_participation
-      Split.redis.hincrby key, 'participant_count', 1
+      @participant_count = Split.redis.hincrby key, 'participant_count', 1
     end
 
     def increment_completion
-      Split.redis.hincrby key, 'completed_count', 1
+       @completed_count = Split.redis.hincrby key, 'completed_count', 1
     end
 
     def control?
@@ -91,6 +93,8 @@ def save
     end
 
     def reset
+      @participant_count = nil
+      @completed_count = nil
       Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0
     end
 

lib/split/configuration.rb

@@ -20,8 +20,80 @@ class Configuration
     attr_accessor :db_failover_allow_parameter_override
     attr_accessor :allow_multiple_experiments
     attr_accessor :enabled
-    attr_accessor :persistence
     attr_accessor :experiments
+    attr_accessor :persistence
+    attr_accessor :algorithm
+
+    def disabled?
+      !enabled
+    end
+
+    def experiment_for(name)
+      if normalized_experiments
+        normalized_experiments[name]
+      end
+    end
+
+    def metrics
+      return @metrics if defined?(@metrics)
+      @metrics = {}
+      if self.experiments
+        self.experiments.each do |key, value|
+          metric_name = value[:metric]
+          if metric_name
+            @metrics[metric_name] ||= []
+            @metrics[metric_name] << Split::Experiment.load_from_configuration(key)
+          end
+        end
+      end
+      @metrics
+    end
+
+    def normalized_experiments
+      if @experiments.nil?
+        nil
+      else
+        experiment_config = {}
+        @experiments.keys.each do | name |
+          experiment_config[name] = {}
+        end
+        @experiments.each do | experiment_name, settings|
+          experiment_config[experiment_name][:variants] = normalize_variants(settings[:variants]) if settings[:variants] 
+        end
+        experiment_config
+      end
+    end
+
+
+    def normalize_variants(variants)
+      given_probability, num_with_probability = variants.inject([0,0]) do |a,v|
+        p, n = a
+        if v.kind_of?(Hash) && v[:percent]
+          [p + v[:percent], n + 1]
+        else
+          a
+        end
+      end
+
+      num_without_probability = variants.length - num_with_probability
+      unassigned_probability = ((100.0 - given_probability) / num_without_probability / 100.0)
+
+      if num_with_probability.nonzero?
+        variants = variants.map do |v|
+          if v.kind_of?(Hash) && v[:name] && v[:percent]
+            { v[:name] => v[:percent] / 100.0 }
+          elsif v.kind_of?(Hash) && v[:name]
+            { v[:name] => unassigned_probability }
+          else
+            { v => unassigned_probability }
+          end
+        end
+        [variants.shift, variants]
+      else
+        variants = variants.dup
+        [variants.shift, variants]
+      end
+    end
 
     def initialize
       @robot_regex = /\b(#{BOTS.keys.join('|')})\b/i
@@ -31,7 +103,9 @@ def initialize
       @db_failover_allow_parameter_override = false
       @allow_multiple_experiments = false
       @enabled = true
+      @experiments = {}
       @persistence = Split::Persistence::SessionAdapter
+      @algorithm = Split::Algorithms::WeightedSample
     end
   end
 end

lib/split/exceptions.rb

@@ -1,3 +1,4 @@
 module Split
   class InvalidPersistenceAdapterError < StandardError; end
+  class ExperimentNotFound < StandardError; end
 end