Skip to content
Browse files

Add tests for control value

  • Loading branch information...
1 parent 7affad3 commit 01571906da38f79b4febb18775d806396dcbee65 Ariel Salomon committed with sideshowbandana Sep 18, 2011
Showing with 85 additions and 0 deletions.
  1. +84 −0 test/experiment/ab_test.rb
  2. +1 −0 test/test_helper.rb
View
84 test/experiment/ab_test.rb
@@ -208,6 +208,90 @@ def test_records_conversion_only_for_participants
assert_equal 100, alts.map(&:converted).sum
end
+ TEST_SIZE = 10000
+
+ def self.should_choose_correct_proportions(delta=1, test_pct=100, control=nil)
+ should "choose correct proportions" do
+ assert_equal TEST_SIZE*(100-test_pct)/100, @results.count(control) if control
+ expected_size = @alternatives.size + (test_pct == 100 || @alternatives.include?(control) ? 0 : 1)
+ assert_equal expected_size, @experiment.alternatives.size
+ @alternatives.delete(control)
+ @alternatives.each do |alt|
+ assert_in_delta TEST_SIZE*test_pct/100/@alternatives.size, @results.count(alt), delta
+ end
+ end
+ end
+
+ def run_test(experiment)
+ (0..TEST_SIZE-1).collect do |hash|
+ index = experiment.send(:hash_to_alternative, hash) # access protected method
+ experiment.alternatives[index].value
+ end
+ end
+
+ context "An experiment with three alternatives" do
+ setup do
+ @experiment = Vanity::Experiment::AbTest.new(Vanity.playground, :test, "test")
+ @alternatives = [1, 2, 3]
+ @experiment.alternatives(*@alternatives)
+ end
+
+ context "with an out-of-set control value" do
+ setup do
+ @experiment.control_value(222)
+ @experiment.test_percent(15)
+ @results = run_test(@experiment)
+ end
+
+ should_choose_correct_proportions(TEST_SIZE/500, 15, 222)
+ end
+
+ context "with an in-set control value" do
+ setup do
+ @experiment.control_value(3)
+ @experiment.test_percent(10)
+ @results = run_test(@experiment)
+ end
+
+ should_choose_correct_proportions(TEST_SIZE/500, 10, 3)
+ end
+
+ context "with no control value" do
+ setup do
+ @results = run_test(@experiment)
+ end
+
+ should_choose_correct_proportions
+ end
+ end
+
+ context "An experiment setting alternatives after control value" do
+ setup do
+ @experiment = Vanity::Experiment::AbTest.new(Vanity.playground, :test, "test")
+ @experiment.test_percent(25)
+ @alternatives = [1, 2, 3]
+ end
+
+ context "with an out-of-set control value" do
+ setup do
+ @experiment.control_value(22)
+ @experiment.alternatives(*@alternatives)
+ @results = run_test(@experiment)
+ end
+
+ should_choose_correct_proportions(TEST_SIZE/500, 25, 22)
+ end
+
+ context "with an in-set control value" do
+ setup do
+ @experiment.control_value(3)
+ @experiment.alternatives(*@alternatives)
+ @results = run_test(@experiment)
+ end
+
+ should_choose_correct_proportions(TEST_SIZE/500, 25, 3)
+ end
+ end
def test_destroy_experiment
new_ab_test :simple do
View
1 test/test_helper.rb
@@ -6,6 +6,7 @@
require "test/unit"
require "mocha"
+require "shoulda"
require "action_controller"
require "action_controller/test_case"
require "action_view/test_case"

0 comments on commit 0157190

Please sign in to comment.
Something went wrong with that request. Please try again.