diff --git a/README.mdown b/README.mdown index 85cd62e4..6c7ee5d8 100644 --- a/README.mdown +++ b/README.mdown @@ -314,12 +314,34 @@ Your code may then track a completion using the metric instead of the experiment name: finished(:conversion) - -You can also create a new metric by instantiating and saving a new Metric object. + +You can also create a new metric by instantiating and saving a new Metric object. Split::Metric.new(:conversion) Split::Metric.save +#### Goals + +You might wish to allow an experiment to have multiple goals. +The API to define goals for an experiment is this: + + ab_test({"link_color" => ["purchase", "refund"]}, "red", "blue") + +or you can you can define them in a configuration file: + + Split.configure do |config| + config.experiments = { + :link_color => { + :alternatives => ["red", "blue"], + :goals => ["purchase", "refund"] + } + } + end + +To complete a goal conversion, you do it like: + + finished({"link_color" => ["purchase"]}) + ### DB failover solution Due to the fact that Redis has no autom. failover mechanism, it's @@ -388,7 +410,7 @@ is configured. ## Outside of a Web Session -Split provides the Helper module to facilitate running experiments inside web sessions. +Split provides the Helper module to facilitate running experiments inside web sessions. Alternatively, you can access the underlying Metric, Trial, Experiment and Alternative objects to conduct experiments that are not tied to a web session. @@ -412,11 +434,11 @@ end ## Algorithms -By default, Split ships with an algorithm that randomly selects from possible alternatives for a traditional a/b test. +By default, Split ships with an algorithm that randomly selects from possible alternatives for a traditional a/b test. -An implementation of a bandit algorithm is also provided. +An implementation of a bandit algorithm is also provided. -Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file. +Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file. ## Extensions diff --git a/lib/split/alternative.rb b/lib/split/alternative.rb index e50e976a..ad6e6a7c 100644 --- a/lib/split/alternative.rb +++ b/lib/split/alternative.rb @@ -19,50 +19,71 @@ def to_s name end + def goals + self.experiment.goals + end + def participant_count - @participant_count ||= Split.redis.hget(key, 'participant_count').to_i + Split.redis.hget(key, 'participant_count').to_i end def participant_count=(count) - @participant_count = count Split.redis.hset(key, 'participant_count', count.to_i) end - def completed_count - @completed_count ||= Split.redis.hget(key, 'completed_count').to_i + def completed_count(goal = nil) + field = set_field(goal) + Split.redis.hget(key, field).to_i + end + + def all_completed_count + if goals.empty? + completed_count + else + goals.inject(completed_count) do |sum, g| + sum + completed_count(g) + end + end end - + def unfinished_count - participant_count - completed_count + participant_count - all_completed_count + end + + def set_field(goal) + field = "completed_count" + field += ":" + goal unless goal.nil? + return field end - def completed_count=(count) - @completed_count = count - Split.redis.hset(key, 'completed_count', count.to_i) + def set_completed_count (count, goal = nil) + field = set_field(goal) + Split.redis.hset(key, field, count.to_i) end def increment_participation - @participant_count = Split.redis.hincrby key, 'participant_count', 1 + Split.redis.hincrby key, 'participant_count', 1 end - def increment_completion - @completed_count = Split.redis.hincrby key, 'completed_count', 1 + def increment_completion(goal = nil) + field = set_field(goal) + Split.redis.hincrby(key, field, 1) end def control? experiment.control.name == self.name end - def conversion_rate + def conversion_rate(goal = nil) return 0 if participant_count.zero? - (completed_count.to_f/participant_count.to_f) + (completed_count(goal).to_f)/participant_count.to_f end def experiment Split::Experiment.find(experiment_name) end - def z_score + def z_score(goal = nil) # CTR_E = the CTR within the experiment split # CTR_C = the CTR within the control split # E = the number of impressions within the experiment split @@ -74,8 +95,9 @@ def z_score return 'N/A' if control.name == alternative.name - ctr_e = alternative.conversion_rate - ctr_c = control.conversion_rate + ctr_e = alternative.conversion_rate(goal) + ctr_c = control.conversion_rate(goal) + e = alternative.participant_count c = control.participant_count @@ -93,9 +115,13 @@ def save end def reset - @participant_count = nil - @completed_count = nil Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0 + unless goals.empty? + goals.each do |g| + field = "completed_count:#{g}" + Split.redis.hset key, field, 0 + end + end end def delete diff --git a/lib/split/configuration.rb b/lib/split/configuration.rb index 7b4b8b50..5ba9aed6 100644 --- a/lib/split/configuration.rb +++ b/lib/split/configuration.rb @@ -48,7 +48,7 @@ def metrics end @metrics end - + def normalized_experiments if @experiments.nil? nil @@ -58,13 +58,13 @@ def normalized_experiments experiment_config[name] = {} end @experiments.each do | experiment_name, settings| - experiment_config[experiment_name][:alternatives] = normalize_alternatives(settings[:alternatives]) if settings[:alternatives] + experiment_config[experiment_name][:alternatives] = normalize_alternatives(settings[:alternatives]) if settings[:alternatives] + experiment_config[experiment_name][:goals] = settings[:goals] if settings[:goals] end experiment_config end end - - + def normalize_alternatives(alternatives) given_probability, num_with_probability = alternatives.inject([0,0]) do |a,v| p, n = a diff --git a/lib/split/dashboard/public/style.css b/lib/split/dashboard/public/style.css index 2afce084..7cb46da4 100644 --- a/lib/split/dashboard/public/style.css +++ b/lib/split/dashboard/public/style.css @@ -160,6 +160,10 @@ body { margin:30px 0; } +.experiment_with_goal { + margin: -32px 0 30px 0; +} + .experiment .experiment-header { background: #f4f4f4; background: -webkit-gradient(linear, left top, left bottom, @@ -177,14 +181,18 @@ body { .experiment h2 { color:#888; - margin: 12px 0 0; + margin: 12px 0 12px 0; font-size: 1em; font-weight:bold; float:left; text-shadow:0 1px 0 rgba(255,255,255,0.8); } -.experiment h2 .version{ +.experiment h2 .goal { + font-style: italic; +} + +.experiment h2 .version { font-style:italic; font-size:0.8em; color:#bbb; diff --git a/lib/split/dashboard/views/_experiment.erb b/lib/split/dashboard/views/_experiment.erb index 1af6686c..3a6c9547 100644 --- a/lib/split/dashboard/views/_experiment.erb +++ b/lib/split/dashboard/views/_experiment.erb @@ -1,20 +1,28 @@ -
+<% unless goal.nil? %> + <% experiment_class = "experiment experiment_with_goal" %> +<% else %> + <% experiment_class = "experiment" %> +<% end %> +

Experiment: <%= experiment.name %> <% if experiment.version > 1 %>v<%= experiment.version %><% end %> + <% unless goal.nil? %>Goal:<%= goal %><% end %>

-
- <%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %> -
" method='post' onclick="return confirmReset()"> - -
-
" method='post' onclick="return confirmDelete()"> - - -
-
+ <% if goal.nil? %> +
+ <%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %> +
" method='post' onclick="return confirmReset()"> + +
+
" method='post' onclick="return confirmDelete()"> + + +
+
+ <% end %>
@@ -27,7 +35,7 @@ - <% total_participants = total_completed = 0 %> + <% total_participants = total_completed = total_unfinished = 0 %> <% experiment.alternatives.each do |alternative| %> - + <% total_participants += alternative.participant_count %> - <% total_completed += alternative.completed_count %> + <% total_unfinished += alternative.unfinished_count %> + <% total_completed += alternative.completed_count(goal) %> <% end %> - + diff --git a/lib/split/dashboard/views/_experiment_with_goal_header.erb b/lib/split/dashboard/views/_experiment_with_goal_header.erb new file mode 100644 index 00000000..24d09bcb --- /dev/null +++ b/lib/split/dashboard/views/_experiment_with_goal_header.erb @@ -0,0 +1,14 @@ +
+
+
+ <%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %> +
" method='post' onclick="return confirmReset()"> + + +
" method='post' onclick="return confirmDelete()"> + + + +
+
+
\ No newline at end of file diff --git a/lib/split/dashboard/views/index.erb b/lib/split/dashboard/views/index.erb index e7804203..b0cf5a60 100644 --- a/lib/split/dashboard/views/index.erb +++ b/lib/split/dashboard/views/index.erb @@ -2,7 +2,14 @@

The list below contains all the registered experiments along with the number of test participants, completed and conversion rate currently in the system.

<% @experiments.each do |experiment| %> - <%= erb :_experiment, :locals => {:experiment => experiment} %> + <% if experiment.goals.empty? %> + <%= erb :_experiment, :locals => {:goal => nil, :experiment => experiment} %> + <% else %> + <%= erb :_experiment_with_goal_header, :locals => {:experiment => experiment} %> + <% experiment.goals.each do |g| %> + <%= erb :_experiment, :locals => {:goal => g, :experiment => experiment} %> + <% end %> + <% end %> <% end %> <% else %>

No experiments have started yet, you need to define them in your code and introduce them to your users.

diff --git a/lib/split/experiment.rb b/lib/split/experiment.rb index 66034ef0..c68c2fcd 100644 --- a/lib/split/experiment.rb +++ b/lib/split/experiment.rb @@ -3,32 +3,37 @@ class Experiment attr_accessor :name attr_writer :algorithm attr_accessor :resettable + attr_accessor :goals - def initialize(name, options = {}) + def initialize(name, options = {}) options = { :resettable => true, }.merge(options) - - @name = name.to_s + + @name = name.to_s @alternatives = options[:alternatives] if !options[:alternatives].nil? - - if !options[:algorithm].nil? + + if !options[:goals].nil? + @goals = options[:goals] + end + + if !options[:algorithm].nil? @algorithm = options[:algorithm].is_a?(String) ? options[:algorithm].constantize : options[:algorithm] end - + if !options[:resettable].nil? @resettable = options[:resettable].is_a?(String) ? options[:resettable] == 'true' : options[:resettable] end - - if !options[:alternative_names].nil? + + if !options[:alternative_names].nil? @alternatives = options[:alternative_names].map do |alternative| Split::Alternative.new(alternative, name) end end - - + + end - + def algorithm @algorithm ||= Split.configuration.algorithm end @@ -44,7 +49,7 @@ def winner nil end end - + def participant_count alternatives.inject(0){|sum,a| sum + a.participant_count} end @@ -65,9 +70,9 @@ def start_time t = Split.redis.hget(:experiment_start_times, @name) Time.parse(t) if t end - + def [](name) - alternatives.find{|a| a.name == name} + alternatives.find{|a| a.name == name} end def alternatives @@ -106,6 +111,14 @@ def key end end + def self.goals_key(name) + "#{name}:goals" + end + + def goals_key + self.class.goals_key(self.name) + end + def finished_key "#{key}:finished" end @@ -125,9 +138,14 @@ def delete reset_winner Split.redis.srem(:experiments, name) Split.redis.del(name) + delete_goals increment_version end + def delete_goals + Split.redis.del(goals_key) + end + def new_record? !Split.redis.exists(name) end @@ -136,10 +154,12 @@ def save if new_record? Split.redis.sadd(:experiments, name) Split.redis.hset(:experiment_start_times, @name, Time.now) - @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name) } + @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} + @goals.reverse.each {|a| Split.redis.lpush(goals_key, a)} unless @goals.nil? else Split.redis.del(name) - @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name) } + @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} + @goals.reverse.each {|a| Split.redis.lpush(goals_key, a)} unless @goals.nil? end config_key = Split::Experiment.experiment_config_key(name) Split.redis.hset(config_key, :resettable, resettable) @@ -147,6 +167,27 @@ def save self end + def self.load_goals_for(name) + if Split.configuration.experiment_for(name) + load_goals_from_configuration_for(name) + else + load_goals_from_redis_for(name) + end + end + + def self.load_goals_from_configuration_for(name) + goals = Split.configuration.experiment_for(name)[:goals] + if goals.nil? + goals = [] + else + goals.flatten + end + end + + def self.load_goals_from_redis_for(name) + Split.redis.lrange(goals_key(name), 0, -1) + end + def self.load_alternatives_for(name) if Split.configuration.experiment_for(name) load_alternatives_from_configuration_for(name) @@ -179,18 +220,20 @@ def self.load_alternatives_from_redis_for(name) def self.load_from_configuration(name) exp_config = Split.configuration.experiment_for(name) || {} - self.new(name, :alternative_names => load_alternatives_for(name), - :resettable => exp_config[:resettable], - :algorithm => exp_config[:algorithm]) + self.new(name, :alternative_names => load_alternatives_for(name), + :goals => load_goals_for(name), + :resettable => exp_config[:resettable], + :algorithm => exp_config[:algorithm]) end def self.load_from_redis(name) exp_config = Split.redis.hgetall(experiment_config_key(name)) self.new(name, :alternative_names => load_alternatives_for(name), - :resettable => exp_config['resettable'], - :algorithm => exp_config['algorithm']) + :goals => load_goals_for(name), + :resettable => exp_config['resettable'], + :algorithm => exp_config['algorithm']) end - + def self.experiment_config_key(name) "experiment_configurations/#{name}" end @@ -219,8 +262,9 @@ def self.find(name) obj end - def self.find_or_create(key, *alternatives) - name = key.to_s.split(':')[0] + def self.find_or_create(label, *alternatives) + experiment_name_with_version, goals = normalize_experiment(label) + name = experiment_name_with_version.to_s.split(':')[0] if alternatives.length == 1 if alternatives[0].is_a? Hash @@ -229,26 +273,40 @@ def self.find_or_create(key, *alternatives) end alts = initialize_alternatives(alternatives, name) + gls = initialize_goals(goals) if Split.redis.exists(name) existing_alternatives = load_alternatives_for(name) - if existing_alternatives == alts.map(&:name) - experiment = self.new(name, :alternative_names => alternatives) + existing_goals = load_goals_for(name) + if existing_alternatives == alts.map(&:name) && existing_goals == gls + experiment = self.new(name, :alternative_names => alternatives, :goals => goals) else - exp = self.new(name, :alternative_names => existing_alternatives) + exp = self.new(name, :alternative_names => existing_alternatives, :goals => goals) exp.reset exp.alternatives.each(&:delete) - experiment = self.new(name, :alternative_names =>alternatives) + exp.delete_goals + experiment = self.new(name, :alternative_names =>alternatives, :goals => goals) experiment.save end else - experiment = self.new(name, :alternative_names => alternatives) + experiment = self.new(name, :alternative_names => alternatives, :goals => goals) experiment.save end return experiment end + def self.normalize_experiment(label) + if Hash === label + experiment_name = label.keys.first + goals = label.values.first + else + experiment_name = label + goals = [] + end + return experiment_name, goals + end + def self.initialize_alternatives(alternatives, name) unless alternatives.all? { |a| Split::Alternative.valid?(a) } @@ -259,5 +317,14 @@ def self.initialize_alternatives(alternatives, name) Split::Alternative.new(alternative, name) end end + + def self.initialize_goals(goals) + raise ArgumentError, 'Goals must be an array' unless valid_goals?(goals) + goals + end + + def self.valid_goals?(goals) + Array === goals rescue false + end end end diff --git a/lib/split/helper.rb b/lib/split/helper.rb index b0a250c1..ed8a76e0 100644 --- a/lib/split/helper.rb +++ b/lib/split/helper.rb @@ -1,14 +1,16 @@ module Split module Helper - def ab_test(experiment_name, control=nil, *alternatives) + def ab_test(experiment_label, control=nil, *alternatives) if RUBY_VERSION.match(/1\.8/) && alternatives.length.zero? && ! control.nil? puts 'WARNING: You should always pass the control alternative through as the second argument with any other alternatives as the third because the order of the hash is not preserved in ruby 1.8' end + experiment_name, goals = normalize_experiment(experiment_label) + begin ret = if Split.configuration.enabled - load_and_start_trial(experiment_name, control, alternatives) + load_and_start_trial(experiment_label, control, alternatives) else control_variable(control) end @@ -49,7 +51,7 @@ def finish_experiment(experiment, options = {:reset => true}) return true else alternative_name = ab_user[experiment.key] - trial = Trial.new(:experiment => experiment, :alternative_name => alternative_name) + trial = Trial.new(:experiment => experiment, :alternative_name => alternative_name, :goals => options[:goals]) trial.complete! if should_reset reset!(experiment) @@ -62,11 +64,12 @@ def finish_experiment(experiment, options = {:reset => true}) def finished(metric_name, options = {:reset => true}) return if exclude_visitor? || Split.configuration.disabled? + metric_name, goals = normalize_experiment(metric_name) experiments = Metric.possible_experiments(metric_name) if experiments.any? experiments.each do |experiment| - finish_experiment(experiment, options) + finish_experiment(experiment, options.merge(:goals => goals)) end end rescue => e @@ -132,18 +135,29 @@ def is_ignored_ip_address? end protected + def normalize_experiment(experiment_label) + if Hash === experiment_label + experiment_name = experiment_label.keys.first + goals = experiment_label.values.first + else + experiment_name = experiment_label + goals = [] + end + return experiment_name, goals + end def control_variable(control) Hash === control ? control.keys.first : control end - def load_and_start_trial(experiment_name, control, alternatives) + def load_and_start_trial(experiment_label, control, alternatives) + experiment_name, goals = normalize_experiment(experiment_label) if control.nil? && alternatives.length.zero? experiment = Experiment.find(experiment_name) raise ExperimentNotFound.new("#{experiment_name} not found") if experiment.nil? else - experiment = Split::Experiment.find_or_create(experiment_name, *([control] + alternatives)) + experiment = Split::Experiment.find_or_create(experiment_label, *([control] + alternatives)) end start_trial( Trial.new(:experiment => experiment) ) diff --git a/lib/split/metric.rb b/lib/split/metric.rb index 5841a60c..6ab76628 100644 --- a/lib/split/metric.rb +++ b/lib/split/metric.rb @@ -64,5 +64,18 @@ def complete! experiment.complete! end end + + private + + def self.normalize_metric(label) + if Hash === label + metric_name = label.keys.first + goals = label.values.first + else + metric_name = label + goals = [] + end + return metric_name, goals + end end end \ No newline at end of file diff --git a/lib/split/trial.rb b/lib/split/trial.rb index b5f1b68b..4f024f99 100644 --- a/lib/split/trial.rb +++ b/lib/split/trial.rb @@ -2,11 +2,13 @@ module Split class Trial attr_accessor :experiment attr_writer :alternative + attr_accessor :goals def initialize(attrs = {}) self.experiment = attrs[:experiment] if !attrs[:experiment].nil? self.alternative = attrs[:alternative] if !attrs[:alternative].nil? self.alternative_name = attrs[:alternative_name] if !attrs[:alternative_name].nil? + self.goals = attrs[:goals] if !attrs[:goals].nil? end def alternative @@ -16,14 +18,20 @@ def alternative end def complete! - alternative.increment_completion if alternative + if alternative + if self.goals.empty? + alternative.increment_completion + else + self.goals.each {|g| alternative.increment_completion(g)} + end + end end def choose! choose record! end - + def record! alternative.increment_participation end @@ -35,7 +43,7 @@ def choose self.alternative = experiment.next_alternative end end - + def alternative_name=(name) self.alternative= self.experiment.alternatives.find{|a| a.name == name } end diff --git a/spec/alternative_spec.rb b/spec/alternative_spec.rb index b7ef1bad..8c25d654 100644 --- a/spec/alternative_spec.rb +++ b/spec/alternative_spec.rb @@ -3,31 +3,51 @@ describe Split::Alternative do + let(:alternative) { + Split::Alternative.new('Basket', 'basket_text') + } + + let(:alternative2) { + Split::Alternative.new('Cart', 'basket_text') + } + + let(:experiment) { + Split::Experiment.find_or_create({"basket_text" => ["purchase", "refund"]}, "Basket", "Cart") + } + + let(:goal1) { "purchase" } + let(:goal2) { "refund" } + + # setup experiment + before do + experiment + end + + it "should have goals" do + alternative.goals.should eql(["purchase", "refund"]) + end + it "should have a name" do - experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.name.should eql('Basket') end it "return only the name" do - experiment = Split::Experiment.new('basket_text', :alternative_names => [{'Basket' => 0.6}, {"Cart" => 0.4}]) - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.name.should eql('Basket') end - + describe 'weights' do - + it "should set the weights" do experiment = Split::Experiment.new('basket_text', :alternative_names => [{'Basket' => 0.6}, {"Cart" => 0.4}]) first = experiment.alternatives[0] first.name.should == 'Basket' first.weight.should == 0.6 - + second = experiment.alternatives[1] second.name.should == 'Cart' - second.weight.should == 0.4 + second.weight.should == 0.4 end - + it "accepts probability on alternatives" do Split.configuration.experiments = { :my_experiment => { @@ -35,19 +55,19 @@ { :name => "control_opt", :percent => 67 }, { :name => "second_opt", :percent => 10 }, { :name => "third_opt", :percent => 23 }, - ], + ] } } experiment = Split::Experiment.find(:my_experiment) first = experiment.alternatives[0] first.name.should == 'control_opt' first.weight.should == 0.67 - + second = experiment.alternatives[1] second.name.should == 'second_opt' second.weight.should == 0.1 end - + # it "accepts probability on some alternatives" do # Split.configuration.experiments[:my_experiment] = { # :alternatives => [ @@ -60,7 +80,7 @@ # should start_experiment(:my_experiment).with({"control_opt" => 0.34}, {"second_opt" => 0.215}, {"third_opt" => 0.23}, {"fourth_opt" => 0.215}) # ab_test :my_experiment # end - # + # # it "allows name param without probability" do # Split.configuration.experiments[:my_experiment] = { # :alternatives => [ @@ -72,113 +92,115 @@ # should start_experiment(:my_experiment).with({"control_opt" => 0.18}, {"second_opt" => 0.18}, {"third_opt" => 0.64}) # ab_test :my_experiment # end - + it "should set the weights from a configuration file" do - + end end it "should have a default participation count of 0" do - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.participant_count.should eql(0) end - it "should have a default completed count of 0" do - alternative = Split::Alternative.new('Basket', 'basket_text') + it "should have a default completed count of 0 for each goal" do alternative.completed_count.should eql(0) + alternative.completed_count(goal1).should eql(0) + alternative.completed_count(goal2).should eql(0) end it "should belong to an experiment" do - experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) - experiment.save - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.experiment.name.should eql(experiment.name) end it "should save to redis" do - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.save Split.redis.exists('basket_text:Basket').should be true end it "should increment participation count" do - experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) - experiment.save - alternative = Split::Alternative.new('Basket', 'basket_text') old_participant_count = alternative.participant_count alternative.increment_participation alternative.participant_count.should eql(old_participant_count+1) - - Split::Alternative.new('Basket', 'basket_text').participant_count.should eql(old_participant_count+1) end - it "should increment completed count" do - experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) - experiment.save - alternative = Split::Alternative.new('Basket', 'basket_text') - old_completed_count = alternative.participant_count + it "should increment completed count for each goal" do + old_default_completed_count = alternative.completed_count + old_completed_count_for_goal1 = alternative.completed_count(goal1) + old_completed_count_for_goal2 = alternative.completed_count(goal2) + alternative.increment_completion - alternative.completed_count.should eql(old_completed_count+1) + alternative.increment_completion(goal1) + alternative.increment_completion(goal2) - Split::Alternative.new('Basket', 'basket_text').completed_count.should eql(old_completed_count+1) + alternative.completed_count.should eql(old_default_completed_count+1) + alternative.completed_count(goal1).should eql(old_completed_count_for_goal1+1) + alternative.completed_count(goal2).should eql(old_completed_count_for_goal2+1) end it "can be reset" do - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.participant_count = 10 - alternative.completed_count = 4 + alternative.set_completed_count(4, goal1) + alternative.set_completed_count(5, goal2) + alternative.set_completed_count(6) alternative.reset alternative.participant_count.should eql(0) + alternative.completed_count(goal1).should eql(0) + alternative.completed_count(goal2).should eql(0) alternative.completed_count.should eql(0) end it "should know if it is the control of an experiment" do - experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) - experiment.save - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.control?.should be_true - alternative = Split::Alternative.new('Cart', 'basket_text') - alternative.control?.should be_false + alternative2.control?.should be_false end describe 'unfinished_count' do it "should be difference between participant and completed counts" do - experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) - experiment.save - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.increment_participation alternative.unfinished_count.should eql(alternative.participant_count) end + + it "should return the correct unfinished_count" do + alternative.participant_count = 10 + alternative.set_completed_count(4, goal1) + alternative.set_completed_count(3, goal2) + alternative.set_completed_count(2) + + alternative.unfinished_count.should eql(1) + end end describe 'conversion rate' do it "should be 0 if there are no conversions" do - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.completed_count.should eql(0) alternative.conversion_rate.should eql(0) end it "calculate conversion rate" do - alternative = Split::Alternative.new('Basket', 'basket_text') alternative.stub(:participant_count).and_return(10) alternative.stub(:completed_count).and_return(4) alternative.conversion_rate.should eql(0.4) + + alternative.stub(:completed_count).with(goal1).and_return(5) + alternative.conversion_rate(goal1).should eql(0.5) + + alternative.stub(:completed_count).with(goal2).and_return(6) + alternative.conversion_rate(goal2).should eql(0.6) end end describe 'z score' do it 'should be zero when the control has no conversions' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') - - alternative = Split::Alternative.new('red', 'link_color') - alternative.z_score.should eql(0) + alternative2.z_score.should eql(0) + alternative2.z_score(goal1).should eql(0) + alternative2.z_score(goal2).should eql(0) end it "should be N/A for the control" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') - control = experiment.control control.z_score.should eql('N/A') + control.z_score(goal1).should eql('N/A') + control.z_score(goal2).should eql('N/A') end end end diff --git a/spec/dashboard_spec.rb b/spec/dashboard_spec.rb index 2d63a3df..5e8c9697 100644 --- a/spec/dashboard_spec.rb +++ b/spec/dashboard_spec.rb @@ -5,6 +5,22 @@ describe Split::Dashboard do include Rack::Test::Methods + let(:link_color) { + Split::Experiment.find_or_create('link_color', 'blue', 'red') + } + + def link(color) + Split::Alternative.new(color, 'link_color') + end + + let(:red_link) { + link("red") + } + + let(:blue_link) { + link("blue") + } + def app @app ||= Split::Dashboard end @@ -15,33 +31,33 @@ def app end it "should reset an experiment" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') + experiment = link_color - red = Split::Alternative.new('red', 'link_color') - blue = Split::Alternative.new('blue', 'link_color') - red.participant_count = 5 - blue.participant_count = 6 + red_link.participant_count = 5 + blue_link.participant_count = 7 + experiment.winner = 'blue' post '/reset/link_color' last_response.should be_redirect - new_red_count = Split::Alternative.new('red', 'link_color').participant_count - new_blue_count = Split::Alternative.new('blue', 'link_color').participant_count + new_red_count = red_link.participant_count + new_blue_count = blue_link.participant_count new_blue_count.should eql(0) new_red_count.should eql(0) + experiment.winner.should be_nil end it "should delete an experiment" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') + experiment = link_color delete '/link_color' last_response.should be_redirect Split::Experiment.find('link_color').should be_nil end it "should mark an alternative as the winner" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') + experiment = link_color experiment.winner.should be_nil post '/link_color', :alternative => 'red' @@ -53,7 +69,7 @@ def app it "should display the start date" do experiment_start_time = Time.parse('2011-07-07') Time.stub(:now => experiment_start_time) - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') + experiment = link_color get '/' @@ -63,7 +79,7 @@ def app it "should handle experiments without a start date" do experiment_start_time = Time.parse('2011-07-07') Time.stub(:now => experiment_start_time) - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') + experiment = link_color Split.redis.hdel(:experiment_start_times, experiment.name) diff --git a/spec/experiment_spec.rb b/spec/experiment_spec.rb index 2ea72075..7396dfec 100644 --- a/spec/experiment_spec.rb +++ b/spec/experiment_spec.rb @@ -4,9 +4,24 @@ require 'time' describe Split::Experiment do + def new_experiment(goals=[]) + Split::Experiment.new('link_color', :alternative_names => ['blue', 'red', 'green'], :goals => goals) + end + + def alternative(color) + Split::Alternative.new(color, 'link_color') + end + + let(:experiment) { + new_experiment + } + + let(:blue) { alternative("blue") } + let(:green) { alternative("green") } + context "with an experiment" do let(:experiment) { Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) } - + it "should have a name" do experiment.name.should eql('basket_text') end @@ -14,15 +29,15 @@ it "should have alternatives" do experiment.alternatives.length.should be 2 end - + it "should have alternatives with correct names" do experiment.alternatives.collect{|a| a.name}.should == ['Basket', 'Cart'] end - + it "should be resettable by default" do experiment.resettable.should be_true end - + it "should save to redis" do experiment.save Split.redis.exists('basket_text').should be true @@ -35,10 +50,10 @@ Split::Experiment.find('basket_text').start_time.should == experiment_start_time end - + it "should save the selected algorithm to redis" do experiment_algorithm = Split::Algorithms::Whiplash - experiment.algorithm = experiment_algorithm + experiment.algorithm = experiment_algorithm experiment.save Split::Experiment.find('basket_text').algorithm.should == experiment_algorithm @@ -60,7 +75,7 @@ Split.redis.exists('basket_text').should be true Split.redis.lrange('basket_text', 0, -1).should eql(['Basket', "Cart"]) end - + describe 'new record?' do it "should know if it hasn't been saved yet" do experiment.new_record?.should be_true @@ -71,18 +86,19 @@ experiment.new_record?.should be_false end end - + describe 'find' do it "should return an existing experiment" do experiment.save - Split::Experiment.find('basket_text').name.should eql('basket_text') + experiment = Split::Experiment.find('basket_text') + experiment.name.should eql('basket_text') end it "should return an existing experiment" do Split::Experiment.find('non_existent_experiment').should be_nil end end - + describe 'control' do it 'should be the first alternative' do experiment.save @@ -90,80 +106,74 @@ end end end + describe 'initialization' do it "should set the algorithm when passed as an option to the initializer" do experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash) experiment.algorithm.should == Split::Algorithms::Whiplash end - + it "should be possible to make an experiment not resettable" do experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :resettable => false) - experiment.resettable.should be_false + experiment.resettable.should be_false end end - + describe 'persistent configuration' do - + it "should persist resettable in redis" do experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :resettable => false) experiment.save - + e = Split::Experiment.find('basket_text') e.should == experiment e.resettable.should be_false - + end - + it "should persist algorithm in redis" do experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash) experiment.save - + e = Split::Experiment.find('basket_text') e.should == experiment e.algorithm.should == Split::Algorithms::Whiplash end end - - - describe 'deleting' do it 'should delete itself' do experiment = Split::Experiment.new('basket_text', :alternative_names => [ 'Basket', "Cart"]) experiment.save experiment.delete - Split.redis.exists('basket_text').should be false - Split::Experiment.find('basket_text').should be_nil + Split.redis.exists('link_color').should be false + Split::Experiment.find('link_color').should be_nil end it "should increment the version" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') experiment.version.should eql(0) experiment.delete experiment.version.should eql(1) end end + describe 'winner' do it "should have no winner initially" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment.winner.should be_nil end it "should allow you to specify a winner" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') + experiment.save experiment.winner = 'red' - - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment.winner.name.should == 'red' end end describe 'reset' do + before { experiment.save } it 'should reset all alternatives' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') - green = Split::Alternative.new('green', 'link_color') experiment.winner = 'green' experiment.next_alternative.name.should eql('green') @@ -171,14 +181,11 @@ experiment.reset - reset_green = Split::Alternative.new('green', 'link_color') - reset_green.participant_count.should eql(0) - reset_green.completed_count.should eql(0) + green.participant_count.should eql(0) + green.completed_count.should eql(0) end it 'should reset the winner' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') - green = Split::Alternative.new('green', 'link_color') experiment.winner = 'green' experiment.next_alternative.name.should eql('green') @@ -190,29 +197,28 @@ end it "should increment the version" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') experiment.version.should eql(0) experiment.reset experiment.version.should eql(1) end end - + describe 'algorithm' do let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') } - + it 'should use the default algorithm if none is specified' do experiment.algorithm.should == Split.configuration.algorithm end - + it 'should use the user specified algorithm for this experiment if specified' do experiment.algorithm = Split::Algorithms::Whiplash experiment.algorithm.should == Split::Algorithms::Whiplash end end - + describe 'next_alternative' do let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') } - + it "should always return the winner if one exists" do green = Split::Alternative.new('green', 'link_color') experiment.winner = 'green' @@ -220,42 +226,42 @@ experiment.next_alternative.name.should eql('green') green.increment_participation - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') experiment.next_alternative.name.should eql('green') end - + it "should use the specified algorithm if a winner does not exist" do Split.configuration.algorithm.should_receive(:choose_alternative).and_return(Split::Alternative.new('green', 'link_color')) experiment.next_alternative.name.should eql('green') end end - + describe 'single alternative' do let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue') } - + it "should always return the color blue" do experiment.next_alternative.name.should eql('blue') - end + end end describe 'changing an existing experiment' do + def same_but_different_alternative + Split::Experiment.find_or_create('link_color', 'blue', 'yellow', 'orange') + end + it "should reset an experiment if it is loaded with different alternatives" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') - blue = Split::Alternative.new('blue', 'link_color') + experiment.save blue.participant_count = 5 - blue.save - same_experiment = Split::Experiment.find_or_create('link_color', 'blue', 'yellow', 'orange') + same_experiment = same_but_different_alternative same_experiment.alternatives.map(&:name).should eql(['blue', 'yellow', 'orange']) - new_blue = Split::Alternative.new('blue', 'link_color') - new_blue.participant_count.should eql(0) + blue.participant_count.should eql(0) end it "should only reset once" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') + experiment.save experiment.version.should eql(0) - same_experiment = Split::Experiment.find_or_create('link_color', 'blue', 'yellow', 'orange') + same_experiment = same_but_different_alternative same_experiment.version.should eql(1) - same_experiment_again = Split::Experiment.find_or_create('link_color', 'blue', 'yellow', 'orange') + same_experiment_again = same_but_different_alternative same_experiment_again.version.should eql(1) end end @@ -268,18 +274,55 @@ end describe 'specifying weights' do - it "should work for a new experiment" do - experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 2 }) + let(:experiment_with_weight) { + Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 2 }) + } - experiment.alternatives.map(&:weight).should == [1, 2] + it "should work for a new experiment" do + experiment_with_weight.alternatives.map(&:weight).should == [1, 2] end it "should work for an existing experiment" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment.save + experiment_with_weight.alternatives.map(&:weight).should == [1, 2] + end + end - same_experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 2 }) - same_experiment.alternatives.map(&:weight).should == [1, 2] + describe "specifying goals" do + let(:experiment) { + new_experiment(["purchase"]) + } + + context "saving experiment" do + def same_but_different_goals + Split::Experiment.find_or_create({'link_color' => ["purchase", "refund"]}, 'blue', 'red', 'green') + end + + before { experiment.save } + + it "can find existing experiment" do + Split::Experiment.find("link_color").name.should eql("link_color") + end + + it "should reset an experiment if it is loaded with different goals" do + same_experiment = same_but_different_goals + Split::Experiment.load_goals_for("link_color").should == ["purchase", "refund"] + end + + end + + it "should have goals" do + experiment.goals.should eql(["purchase"]) + end + + context "find or create experiment" do + it "should have correct goals" do + experiment = Split::Experiment.find_or_create({'link_color3' => ["purchase", "refund"]}, 'blue', 'red', 'green') + experiment.goals.should == ["purchase", "refund"] + experiment = Split::Experiment.find_or_create('link_color3', 'blue', 'red', 'green') + experiment.goals.should == [] + end end end + end diff --git a/spec/helper_spec.rb b/spec/helper_spec.rb index 05b9720a..a14c7369 100755 --- a/spec/helper_spec.rb +++ b/spec/helper_spec.rb @@ -5,6 +5,10 @@ describe Split::Helper do include Split::Helper + let(:experiment) { + Split::Experiment.find_or_create('link_color', 'blue', 'red') + } + describe "ab_test" do it "should not raise an error when passed strings for alternatives" do @@ -19,13 +23,20 @@ lambda { ab_test('xyz', :a, :b, :c) }.should raise_error(ArgumentError) end + it "should not raise error when passed an array for experiment" do + lambda { ab_test({'link_color' => ["purchase", "refund"]}, 'blue', 'red') }.should_not raise_error + end + + it "should raise the appropriate error when passed string for experiment" do + lambda { ab_test({'link_color' => "purchase"}, 'blue', 'red') }.should raise_error(ArgumentError) + end + it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do ab_test('link_color', 'blue', 'red') ['red', 'blue'].should include(ab_user['link_color']) end it "should increment the participation counter after assignment to a new user" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') previous_red_count = Split::Alternative.new('red', 'link_color').participant_count previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count @@ -39,21 +50,18 @@ end it "should return the given alternative for an existing user" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative = ab_test('link_color', 'blue', 'red') repeat_alternative = ab_test('link_color', 'blue', 'red') alternative.should eql repeat_alternative end it 'should always return the winner if one is present' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment.winner = "orange" ab_test('link_color', 'blue', 'red').should == 'orange' end it "should allow the alternative to be force by passing it in the params" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') @params = {'link_color' => 'blue'} alternative = ab_test('link_color', 'blue', 'red') alternative.should eql('blue') @@ -107,7 +115,6 @@ end it "should not over-write a finished key when an experiment is on a later version" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment.increment_version ab_user = { experiment.key => 'blue', experiment.finished_key => true } finshed_session = ab_user.dup @@ -132,12 +139,12 @@ end it "should set experiment's finished key if reset is false" do - finished(@experiment_name, :reset => false) + finished(@experiment_name, {:reset => false}) ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true) end it 'should not increment the counter if reset is false and the experiment has been already finished' do - 2.times { finished(@experiment_name, :reset => false) } + 2.times { finished(@experiment_name, {:reset => false}) } new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count new_completion_count.should eql(@previous_completion_count + 1) end @@ -150,7 +157,7 @@ it "should not clear out the users session if reset is false" do ab_user.should eql(@experiment.key => @alternative_name) - finished(@experiment_name, :reset => false) + finished(@experiment_name, {:reset => false}) ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true) end @@ -257,12 +264,12 @@ def should_finish_experiment(experiment_name, should_finish=true) finished :my_metric, :reset => false ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true) end + end end describe 'conversions' do it 'should return a conversion rate for an alternative' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') previous_convertion_rate = Split::Alternative.new(alternative_name, 'link_color').conversion_rate @@ -282,13 +289,11 @@ def should_finish_experiment(experiment_name, should_finish=true) describe 'ab_test' do it 'should return the control' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative = ab_test('link_color', 'blue', 'red') alternative.should eql experiment.control.name end it "should not increment the participation count" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') previous_red_count = Split::Alternative.new('red', 'link_color').participant_count previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count @@ -304,7 +309,6 @@ def should_finish_experiment(experiment_name, should_finish=true) describe 'finished' do it "should not increment the completed count" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count @@ -328,13 +332,11 @@ def should_finish_experiment(experiment_name, should_finish=true) describe 'ab_test' do it 'should return the control' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative = ab_test('link_color', 'blue', 'red') alternative.should eql experiment.control.name end it "should not increment the participation count" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') previous_red_count = Split::Alternative.new('red', 'link_color').participant_count previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count @@ -350,7 +352,6 @@ def should_finish_experiment(experiment_name, should_finish=true) describe 'finished' do it "should not increment the completed count" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count @@ -366,14 +367,12 @@ def should_finish_experiment(experiment_name, should_finish=true) describe 'versioned experiments' do it "should use version zero if no version is present" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') experiment.version.should eql(0) ab_user.should eql({'link_color' => alternative_name}) end it "should save the version of the experiment to the session" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment.reset experiment.version.should eql(1) alternative_name = ab_test('link_color', 'blue', 'red') @@ -381,7 +380,6 @@ def should_finish_experiment(experiment_name, should_finish=true) end it "should load the experiment even if the version is not 0" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment.reset experiment.version.should eql(1) alternative_name = ab_test('link_color', 'blue', 'red') @@ -391,7 +389,6 @@ def should_finish_experiment(experiment_name, should_finish=true) end it "should reset the session of a user on an older version of the experiment" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') ab_user.should eql({'link_color' => alternative_name}) alternative = Split::Alternative.new(alternative_name, 'link_color') @@ -409,7 +406,6 @@ def should_finish_experiment(experiment_name, should_finish=true) end it "should cleanup old versions of experiments from the session" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') ab_user.should eql({'link_color' => alternative_name}) alternative = Split::Alternative.new(alternative_name, 'link_color') @@ -425,7 +421,6 @@ def should_finish_experiment(experiment_name, should_finish=true) end it "should only count completion of users on the current version" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') ab_user.should eql({'link_color' => alternative_name}) alternative = Split::Alternative.new(alternative_name, 'link_color') @@ -577,25 +572,47 @@ def should_finish_experiment(experiment_name, should_finish=true) context "with preloaded config" do before { Split.configuration.experiments = {}} - + it "pulls options from config file" do Split.configuration.experiments[:my_experiment] = { :alternatives => [ "control_opt", "other_opt" ], + :goals => ["goal1", "goal2"] } ab_test :my_experiment Split::Experiment.find(:my_experiment).alternative_names.should == [ "control_opt", "other_opt" ] + Split::Experiment.find(:my_experiment).goals.should == [ "goal1", "goal2" ] end - + it "can be called multiple times" do Split.configuration.experiments[:my_experiment] = { :alternatives => [ "control_opt", "other_opt" ], + :goals => ["goal1", "goal2"] } 5.times { ab_test :my_experiment } experiment = Split::Experiment.find(:my_experiment) experiment.alternative_names.should == [ "control_opt", "other_opt" ] + experiment.goals.should == [ "goal1", "goal2" ] experiment.participant_count.should == 1 end - + + it "accepts multiple goals" do + Split.configuration.experiments[:my_experiment] = { + :alternatives => [ "control_opt", "other_opt" ], + :goals => [ "goal1", "goal2", "goal3" ] + } + ab_test :my_experiment + experiment = Split::Experiment.find(:my_experiment) + experiment.goals.should == [ "goal1", "goal2", "goal3" ] + end + + it "allow specifying goals to be optional" do + Split.configuration.experiments[:my_experiment] = { + :alternatives => [ "control_opt", "other_opt" ] + } + experiment = Split::Experiment.find(:my_experiment) + experiment.goals.should == [] + end + it "accepts multiple alternatives" do Split.configuration.experiments[:my_experiment] = { :alternatives => [ "control_opt", "second_opt", "third_opt" ], @@ -604,7 +621,7 @@ def should_finish_experiment(experiment_name, should_finish=true) experiment = Split::Experiment.find(:my_experiment) experiment.alternative_names.should == [ "control_opt", "second_opt", "third_opt" ] end - + it "accepts probability on alternatives" do Split.configuration.experiments[:my_experiment] = { :alternatives => [ @@ -616,9 +633,9 @@ def should_finish_experiment(experiment_name, should_finish=true) ab_test :my_experiment experiment = Split::Experiment.find(:my_experiment) experiment.alternatives.collect{|a| [a.name, a.weight]}.should == [['control_opt', 0.67], ['second_opt', 0.1], ['third_opt', 0.23]] - + end - + it "accepts probability on some alternatives" do Split.configuration.experiments[:my_experiment] = { :alternatives => [ @@ -634,7 +651,7 @@ def should_finish_experiment(experiment_name, should_finish=true) names_and_weights.should == [['control_opt', 0.34], ['second_opt', 0.215], ['third_opt', 0.23], ['fourth_opt', 0.215]] names_and_weights.inject(0){|sum, nw| sum + nw[1]}.should == 1.0 end - + it "allows name param without probability" do Split.configuration.experiments[:my_experiment] = { :alternatives => [ @@ -667,7 +684,6 @@ def should_finish_experiment(experiment_name, should_finish=true) end it 'should handle multiple experiments correctly' do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') experiment2 = Split::Experiment.find_or_create('link_color2', 'blue', 'red') alternative_name = ab_test('link_color', 'blue', 'red') alternative_name2 = ab_test('link_color2', 'blue', 'red') @@ -677,4 +693,43 @@ def should_finish_experiment(experiment_name, should_finish=true) alt.unfinished_count.should eq(0) end end + + context "with goals" do + before do + @experiment = {'link_color' => ["purchase", "refund"]} + @alternatives = ['blue', 'red'] + @experiment_name, @goals = normalize_experiment(@experiment) + @goal1 = @goals[0] + @goal2 = @goals[1] + end + + it "should normalize experiment" do + @experiment_name.should eql("link_color") + @goals.should eql(["purchase", "refund"]) + end + + describe "ab_test" do + it "should allow experiment goals interface as a singel hash" do + ab_test(@experiment, *@alternatives) + experiment = Split::Experiment.find('link_color') + experiment.goals.should eql(['purchase', "refund"]) + end + end + + describe "finished" do + before do + @alternative_name = ab_test(@experiment, *@alternatives) + end + + it "should increment the counter for the specified-goal completed alternative" do + @previous_completion_count_for_goal1 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1) + @previous_completion_count_for_goal2 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2) + finished({"link_color" => ["purchase"]}) + new_completion_count_for_goal1 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1) + new_completion_count_for_goal1.should eql(@previous_completion_count_for_goal1 + 1) + new_completion_count_for_goal2 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2) + new_completion_count_for_goal2.should eql(@previous_completion_count_for_goal2) + end + end + end end
Finish
@@ -38,23 +46,23 @@ <%= alternative.participant_count %> <%= alternative.unfinished_count %><%= alternative.completed_count %><%= alternative.completed_count(goal) %> - <%= number_to_percentage(alternative.conversion_rate) %>% - <% if experiment.control.conversion_rate > 0 && !alternative.control? %> - <% if alternative.conversion_rate > experiment.control.conversion_rate %> + <%= number_to_percentage(alternative.conversion_rate(goal)) %>% + <% if experiment.control.conversion_rate(goal) > 0 && !alternative.control? %> + <% if alternative.conversion_rate(goal) > experiment.control.conversion_rate(goal) %> - +<%= number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1) %>% + +<%= number_to_percentage((alternative.conversion_rate(goal)/experiment.control.conversion_rate(goal))-1) %>% - <% elsif alternative.conversion_rate < experiment.control.conversion_rate %> + <% elsif alternative.conversion_rate(goal) < experiment.control.conversion_rate(goal) %> - <%= number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1) %>% + <%= number_to_percentage((alternative.conversion_rate(goal)/experiment.control.conversion_rate(goal))-1) %>% <% end %> <% end %> - <%= confidence_level(alternative.z_score) %> + <%= confidence_level(alternative.z_score(goal)) %> <% if experiment.winner %> @@ -73,13 +81,14 @@
Totals <%= total_participants %><%= total_participants - total_completed %><%= total_unfinished %> <%= total_completed %> N/A N/A