Permalink
Browse files

Bucket strategy with specs

  • Loading branch information...
1 parent c2aedaf commit d3f1c8b024e9256c37f4b29a94c13bf1059d3193 Justin Jones committed Dec 5, 2011
Showing with 113 additions and 4 deletions.
  1. +2 −0 lib/trebuchet.rb
  2. +0 −3 lib/trebuchet/feature.rb
  3. +2 −1 lib/trebuchet/strategy.rb
  4. +23 −0 lib/trebuchet/strategy/bucket.rb
  5. +86 −0 spec/bucket_strategy_spec.rb
View
@@ -63,6 +63,8 @@ def launch?(feature)
require 'trebuchet/strategy/default'
require 'trebuchet/strategy/percentage'
require 'trebuchet/strategy/user_id'
+require 'trebuchet/strategy/bucket'
require 'trebuchet/strategy/custom'
+require 'trebuchet/strategy/invalid'
require 'trebuchet/strategy/multiple'
require 'trebuchet/action_controller'
@@ -42,9 +42,6 @@ def aim(strategy_name, options = nil)
def as_json(options = {})
{:name => @name, :strategy => strategy}
- # .tap do |h|
- # h[:strategy] = strategy if valid?
- # end
end
private
@@ -29,7 +29,8 @@ def self.name_class_map
[:users, UserId],
[:default, Default],
[:custom, Custom],
- [:multiple, Multiple]
+ [:multiple, Multiple],
+ [:bucket, Bucket]
]
end
@@ -0,0 +1,23 @@
+require 'digest/sha1'
+
+class Trebuchet::Strategy::Bucket < Trebuchet::Strategy::Base
+
+ attr_reader :bucket, :bucket_count
+
+ def initialize(options = {})
+ if (n = options).is_a?(Integer)
+ options = {}
+ options[:bucket] = n
+ end
+ @experiment = options[:experiment] || ""
+ @bucket = options[:bucket]
+ @bucket_count = options[:bucket_count] || 10
+ end
+
+ def launch_at?(user)
+ # must hash feature name and user id together to ensure uniform distribution
+ b = Digest::SHA1.hexdigest("experiment: #{@experiment.downcase} user: #{user.id}").to_i(16) % bucket_count
+ !!(b + 1 == @bucket) # is user in this bucket?
+ end
+
+end
@@ -0,0 +1,86 @@
+require 'spec_helper'
+
+describe Trebuchet::Strategy::Bucket do
+
+ before do
+ @feature_name = 'Button Color'
+ end
+
+ it "should match a user in a bucket" do
+ Trebuchet.aim(@feature_name, :bucket, 1)
+ strategy = Trebuchet.feature(@feature_name).strategy
+ # these values just happen to hash for the algorithm and feature name
+ strategy.launch_at?(User.new(4)).should be_true
+ strategy.launch_at?(User.new(5)).should be_false
+ end
+
+ it "should adjust the number of buckets" do
+ Trebuchet.aim(@feature_name, :bucket, {:bucket => 1, :bucket_count => 3})
+ strategy = Trebuchet.feature(@feature_name).strategy
+ strategy.bucket_count.should == 3
+ strategy.bucket.should == 1
+ strategy.launch_at?(User.new(4)).should be_true
+ Trebuchet.aim(@feature_name, :bucket, {:bucket => 2, :bucket_count => 3})
+ Trebuchet.feature(@feature_name).strategy.launch_at?(User.new(4)).should be_false
+ end
+
+ it "should be mutually exclusive within experiments" do
+ strategies = (1..10).map do |i|
+ Trebuchet.aim(@feature_name, :bucket, i)
+ Trebuchet.feature(@feature_name).strategy
+ end
+ user_ids = (1..100).to_a
+ launches = strategies.map do |strategy|
+ user_ids.select {|user_id| strategy.launch_at?(User.new(user_id))}
+ end
+ occurrences = user_ids.map do |user_id|
+ launches.select{|l| l.include?(user_id)}.size
+ end
+ # no user should be in more than one bucket
+ occurrences.select{|i| i > 1}.size.should == 0
+ # each user should be in one bucket
+ occurrences.select{|i| i < 1}.size.should == 0
+ end
+
+ it "should distribute users evenly" do
+ Trebuchet.aim(@feature_name, :bucket, 5)
+ strategy = Trebuchet.feature(@feature_name).strategy
+ user_ids = (1..10_000).to_a
+ launches = user_ids.map {|user_id| strategy.launch_at?(User.new(user_id))}
+ # total should be around 10%
+ launch_count = launches.select{|l| l == true}.size
+ (launch_count * 100 / user_ids.size).round.should == 10
+ end
+
+ it "should have low overlap between experiments" do
+ other_feature_name = 'Eternal Youth'
+ another_feature_name = 'Invisibility'
+ Trebuchet.aim(@feature_name, :bucket, 3)
+ Trebuchet.aim(other_feature_name, :bucket, 5)
+ Trebuchet.aim(another_feature_name, :bucket, 7)
+ strategies = [
+ Trebuchet.feature(@feature_name).strategy,
+ Trebuchet.feature(other_feature_name).strategy,
+ Trebuchet.feature(another_feature_name).strategy
+ ]
+ # find out which users match each strategy
+ user_ids = (1..10_000).to_a
+ launches = strategies.map do |strategy|
+ user_ids.select {|user_id| strategy.launch_at?(User.new(user_id))}
+ end
+ # intersect each set with the next
+ overlaps = []
+ (0..launches.size).each do |i|
+ j = (i + 1) % launches.size
+ overlaps << launches[i] & launches[j]
+ end
+ # each group should have about 10% overlap
+ ((9..11) === (overlaps[0].size * 100 / user_ids.size).round).should be_true
+ ((9..11) === (overlaps[1].size * 100 / user_ids.size).round).should be_true
+ ((9..11) === (overlaps[2].size * 100 / user_ids.size).round).should be_true
+ # 1% or fewer of users should be in all three groups
+ total_overlap = (launches[0] & launches[1] & launches[2])
+ (total_overlap.size * 100 / user_ids.size).round.should < 2
+ end
+
+end

0 comments on commit d3f1c8b

Please sign in to comment.