diff --git a/README.mdown b/README.mdown index 008b0f60..c634d1e5 100644 --- a/README.mdown +++ b/README.mdown @@ -264,7 +264,7 @@ in a hash or a configuration file: config.experiments = YAML.load_file "config/experiments.yml" end -This hash can control your experiment's variants, weights, and if the +This hash can control your experiment's variants, weights, algorithm and if the experiment resets once finished: Split.configure do |config| @@ -274,6 +274,7 @@ experiment resets once finished: :resettable => false, }, :my_second_experiment => { + :algorithm => 'Split::Algorithms::Whiplash', :variants => [ { :name => "a", :percent => 67 }, { :name => "b", :percent => 33 }, @@ -310,6 +311,11 @@ Your code may then track a completion using the metric instead of the experiment name: finished(:conversion) + +You can also create a new metric by instantiating and saving a new Metric object. + + Split::Metric.new(:conversion) + Split::Metric.save ### DB failover solution @@ -377,6 +383,38 @@ Split.redis.namespace = "split:blog" We recommend sticking this in your initializer somewhere after Redis is configured. +## Outside of a Web Session + +Split provides the Helper module to facilitate running experiments inside web sessions. + +Alternatively, you can access the underlying Metric, Trial, Experiment and Alternative objects to +conduct experiments that are not tied to a web session. + +```ruby +# create a new experiment +experiment = Split::Experiment.find_or_create('color', 'red', 'blue') +# create a new trial +trial = Trial.new(:experiment => experiment) +# run trial +trial.choose! +# get the result, returns either red or blue +trial.alternative.name + +# if the goal has been achieved, increment the successful completions for this alternative. +if goal_acheived? + trial.complete! +end + +``` + +## Algorithms + +By default, Split ships with an algorithm that randomly selects from possible alternatives for a traditional a/b test. + +An implementation of a bandit algorithm is also provided. + +Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file. + ## Extensions - [Split::Export](http://github.com/andrew/split-export) - easily export ab test data out of Split diff --git a/lib/split.rb b/lib/split.rb index e3b9649b..ada52dcc 100755 --- a/lib/split.rb +++ b/lib/split.rb @@ -1,4 +1,4 @@ -%w[experiment alternative helper version configuration persistence exceptions].each do |f| +%w[algorithms extensions metric trial experiment alternative helper version configuration persistence exceptions].each do |f| require "split/#{f}" end diff --git a/lib/split/algorithms.rb b/lib/split/algorithms.rb new file mode 100644 index 00000000..f7adf367 --- /dev/null +++ b/lib/split/algorithms.rb @@ -0,0 +1,3 @@ +%w[weighted_sample whiplash].each do |f| + require "split/algorithms/#{f}" +end \ No newline at end of file diff --git a/lib/split/algorithms/weighted_sample.rb b/lib/split/algorithms/weighted_sample.rb new file mode 100644 index 00000000..055d440d --- /dev/null +++ b/lib/split/algorithms/weighted_sample.rb @@ -0,0 +1,17 @@ +module Split + module Algorithms + module WeightedSample + def self.choose_alternative(experiment) + weights = experiment.alternatives.map(&:weight) + + total = weights.inject(:+) + point = rand * total + + experiment.alternatives.zip(weights).each do |n,w| + return n if w >= point + point -= w + end + end + end + end +end \ No newline at end of file diff --git a/lib/split/algorithms/whiplash.rb b/lib/split/algorithms/whiplash.rb new file mode 100644 index 00000000..f17742bd --- /dev/null +++ b/lib/split/algorithms/whiplash.rb @@ -0,0 +1,35 @@ +# A multi-armed bandit implementation inspired by +# @aaronsw and victorykit/whiplash +require 'simple-random' + +module Split + module Algorithms + module Whiplash + def self.choose_alternative(experiment) + experiment[best_guess(experiment.alternatives)] + end + + private + + def self.arm_guess(participants, completions) + a = [participants, 0].max + b = [participants-completions, 0].max + s = SimpleRandom.new; s.set_seed; s.beta(a+fairness_constant, b+fairness_constant) + end + + def self.best_guess(alternatives) + guesses = {} + alternatives.each do |alternative| + guesses[alternative.name] = arm_guess(alternative.participant_count, alternative.completed_count) + end + gmax = guesses.values.max + best = guesses.keys.select {|name| guesses[name] == gmax } + return best.sample + end + + def self.fairness_constant + 7 + end + end + end +end \ No newline at end of file diff --git a/lib/split/alternative.rb b/lib/split/alternative.rb index 5828cc49..e50e976a 100644 --- a/lib/split/alternative.rb +++ b/lib/split/alternative.rb @@ -20,15 +20,16 @@ def to_s end def participant_count - Split.redis.hget(key, 'participant_count').to_i + @participant_count ||= Split.redis.hget(key, 'participant_count').to_i end def participant_count=(count) + @participant_count = count Split.redis.hset(key, 'participant_count', count.to_i) end def completed_count - Split.redis.hget(key, 'completed_count').to_i + @completed_count ||= Split.redis.hget(key, 'completed_count').to_i end def unfinished_count @@ -36,15 +37,16 @@ def unfinished_count end def completed_count=(count) + @completed_count = count Split.redis.hset(key, 'completed_count', count.to_i) end def increment_participation - Split.redis.hincrby key, 'participant_count', 1 + @participant_count = Split.redis.hincrby key, 'participant_count', 1 end def increment_completion - Split.redis.hincrby key, 'completed_count', 1 + @completed_count = Split.redis.hincrby key, 'completed_count', 1 end def control? @@ -91,6 +93,8 @@ def save end def reset + @participant_count = nil + @completed_count = nil Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0 end diff --git a/lib/split/configuration.rb b/lib/split/configuration.rb index e1aaa978..218f2c27 100644 --- a/lib/split/configuration.rb +++ b/lib/split/configuration.rb @@ -20,8 +20,80 @@ class Configuration attr_accessor :db_failover_allow_parameter_override attr_accessor :allow_multiple_experiments attr_accessor :enabled - attr_accessor :persistence attr_accessor :experiments + attr_accessor :persistence + attr_accessor :algorithm + + def disabled? + !enabled + end + + def experiment_for(name) + if normalized_experiments + normalized_experiments[name] + end + end + + def metrics + return @metrics if defined?(@metrics) + @metrics = {} + if self.experiments + self.experiments.each do |key, value| + metric_name = value[:metric] + if metric_name + @metrics[metric_name] ||= [] + @metrics[metric_name] << Split::Experiment.load_from_configuration(key) + end + end + end + @metrics + end + + def normalized_experiments + if @experiments.nil? + nil + else + experiment_config = {} + @experiments.keys.each do | name | + experiment_config[name] = {} + end + @experiments.each do | experiment_name, settings| + experiment_config[experiment_name][:variants] = normalize_variants(settings[:variants]) if settings[:variants] + end + experiment_config + end + end + + + def normalize_variants(variants) + given_probability, num_with_probability = variants.inject([0,0]) do |a,v| + p, n = a + if v.kind_of?(Hash) && v[:percent] + [p + v[:percent], n + 1] + else + a + end + end + + num_without_probability = variants.length - num_with_probability + unassigned_probability = ((100.0 - given_probability) / num_without_probability / 100.0) + + if num_with_probability.nonzero? + variants = variants.map do |v| + if v.kind_of?(Hash) && v[:name] && v[:percent] + { v[:name] => v[:percent] / 100.0 } + elsif v.kind_of?(Hash) && v[:name] + { v[:name] => unassigned_probability } + else + { v => unassigned_probability } + end + end + [variants.shift, variants] + else + variants = variants.dup + [variants.shift, variants] + end + end def initialize @robot_regex = /\b(#{BOTS.keys.join('|')})\b/i @@ -31,7 +103,9 @@ def initialize @db_failover_allow_parameter_override = false @allow_multiple_experiments = false @enabled = true + @experiments = {} @persistence = Split::Persistence::SessionAdapter + @algorithm = Split::Algorithms::WeightedSample end end end diff --git a/lib/split/exceptions.rb b/lib/split/exceptions.rb index ea445682..f59995a7 100644 --- a/lib/split/exceptions.rb +++ b/lib/split/exceptions.rb @@ -1,3 +1,4 @@ module Split class InvalidPersistenceAdapterError < StandardError; end + class ExperimentNotFound < StandardError; end end \ No newline at end of file diff --git a/lib/split/experiment.rb b/lib/split/experiment.rb index 27643a07..d166c2c7 100644 --- a/lib/split/experiment.rb +++ b/lib/split/experiment.rb @@ -1,12 +1,40 @@ module Split class Experiment attr_accessor :name + attr_writer :algorithm + attr_accessor :resettable - def initialize(name, *alternative_names) - @name = name.to_s - @alternatives = alternative_names.map do |alternative| - Split::Alternative.new(alternative, name) - end + def initialize(name, options = {}) + options = { + :resettable => true, + }.merge(options) + + @name = name.to_s + @alternatives = options[:alternatives] if !options[:alternatives].nil? + + if !options[:algorithm].nil? + @algorithm = options[:algorithm].is_a?(String) ? options[:algorithm].constantize : options[:algorithm] + end + + if !options[:resettable].nil? + @resettable = options[:resettable].is_a?(String) ? options[:resettable] == 'true' : options[:resettable] + end + + if !options[:alternative_names].nil? + @alternatives = options[:alternative_names].map do |alternative| + Split::Alternative.new(alternative, name) + end + end + + + end + + def algorithm + @algorithm ||= Split.configuration.algorithm + end + + def ==(obj) + self.name == obj.name end def winner @@ -16,6 +44,10 @@ def winner nil end end + + def participant_count + alternatives.inject(0){|sum,a| sum + a.participant_count} + end def control alternatives.first @@ -33,6 +65,10 @@ def start_time t = Split.redis.hget(:experiment_start_times, @name) Time.parse(t) if t end + + def [](name) + alternatives.find{|a| a.name == name} + end def alternatives @alternatives.dup @@ -47,14 +83,10 @@ def next_alternative end def random_alternative - weights = alternatives.map(&:weight) - - total = weights.inject(:+) - point = rand * total - - alternatives.zip(weights).each do |n,w| - return n if w >= point - point -= w + if alternatives.length > 1 + Split.configuration.algorithm.choose_alternative(self) + else + alternatives.first end end @@ -78,6 +110,10 @@ def finished_key "#{key}:finished" end + def resettable? + resettable + end + def reset alternatives.each(&:reset) reset_winner @@ -105,9 +141,30 @@ def save Split.redis.del(name) @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name) } end + config_key = Split::Experiment.experiment_config_key(name) + Split.redis.hset(config_key, :resettable, resettable) + Split.redis.hset(config_key, :algorithm, algorithm.to_s) + self end def self.load_alternatives_for(name) + if Split.configuration.experiment_for(name) + load_alternatives_from_configuration_for(name) + else + load_alternatives_from_redis_for(name) + end + end + + def self.load_alternatives_from_configuration_for(name) + alts = Split.configuration.experiment_for(name)[:variants] + if alts.is_a?(Hash) + alts.keys + else + alts.flatten + end + end + + def self.load_alternatives_from_redis_for(name) case Split.redis.type(name) when 'set' # convert legacy sets to lists alts = Split.redis.smembers(name) @@ -119,14 +176,46 @@ def self.load_alternatives_for(name) end end + def self.load_from_configuration(name) + exp_config = Split.configuration.experiment_for(name) || {} + self.new(name, :alternative_names => load_alternatives_for(name), + :resettable => exp_config[:resettable], + :algorithm => exp_config[:algorithm]) + end + + def self.load_from_redis(name) + exp_config = Split.redis.hgetall(experiment_config_key(name)) + self.new(name, :alternative_names => load_alternatives_for(name), + :resettable => exp_config['resettable'], + :algorithm => exp_config['algorithm']) + end + + def self.experiment_config_key(name) + "experiment_configurations/#{name}" + end + def self.all - Array(Split.redis.smembers(:experiments)).map {|e| find(e)} + Array(all_experiment_names_from_redis + all_experiment_names_from_configuration).map {|e| find(e)} end + def self.all_experiment_names_from_redis + Split.redis.smembers(:experiments) + end + + def self.all_experiment_names_from_configuration + Split.configuration.experiments ? Split.configuration.experiments.keys : [] + end + + def self.find(name) - if Split.redis.exists(name) - self.new(name, *load_alternatives_for(name)) + if Split.configuration.experiment_for(name) + obj = load_from_configuration(name) + elsif Split.redis.exists(name) + obj = load_from_redis(name) + else + obj = nil end + obj end def self.find_or_create(key, *alternatives) @@ -135,8 +224,6 @@ def self.find_or_create(key, *alternatives) if alternatives.length == 1 if alternatives[0].is_a? Hash alternatives = alternatives[0].map{|k,v| {k => v} } - else - raise ArgumentError, 'You must declare at least 2 alternatives' end end @@ -145,16 +232,16 @@ def self.find_or_create(key, *alternatives) if Split.redis.exists(name) existing_alternatives = load_alternatives_for(name) if existing_alternatives == alts.map(&:name) - experiment = self.new(name, *alternatives) + experiment = self.new(name, :alternative_names => alternatives) else - exp = self.new(name, *existing_alternatives) + exp = self.new(name, :alternative_names => existing_alternatives) exp.reset exp.alternatives.each(&:delete) - experiment = self.new(name, *alternatives) + experiment = self.new(name, :alternative_names =>alternatives) experiment.save end else - experiment = self.new(name, *alternatives) + experiment = self.new(name, :alternative_names => alternatives) experiment.save end return experiment diff --git a/lib/split/extensions.rb b/lib/split/extensions.rb new file mode 100644 index 00000000..4d13fd7c --- /dev/null +++ b/lib/split/extensions.rb @@ -0,0 +1,3 @@ +%w[array string].each do |f| + require "split/extensions/#{f}" +end \ No newline at end of file diff --git a/lib/split/extensions/array.rb b/lib/split/extensions/array.rb new file mode 100644 index 00000000..6e56dc3d --- /dev/null +++ b/lib/split/extensions/array.rb @@ -0,0 +1,4 @@ +class Array + # maintain backwards compatibility with 1.8.7 + alias_method :sample, :choice unless method_defined?(:sample) +end \ No newline at end of file diff --git a/lib/split/extensions/string.rb b/lib/split/extensions/string.rb new file mode 100644 index 00000000..20eba92a --- /dev/null +++ b/lib/split/extensions/string.rb @@ -0,0 +1,15 @@ +class String + # Constatntize is often provided by ActiveSupport, but ActiveSupport is not a dependency of Split. + unless method_defined?(:constantize) + def constantize + names = self.split('::') + names.shift if names.empty? || names.first.empty? + + constant = Object + names.each do |name| + constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name) + end + constant + end + end +end \ No newline at end of file diff --git a/lib/split/helper.rb b/lib/split/helper.rb index 33697c52..0f9860e0 100644 --- a/lib/split/helper.rb +++ b/lib/split/helper.rb @@ -1,20 +1,30 @@ module Split module Helper + def ab_test(experiment_name, control=nil, *alternatives) - if control.nil? && alternatives.length.zero? - experiment_config = Split.configuration.experiments[experiment_name] - unless experiment_config.nil? - control, alternatives = normalize_variants experiment_config[:variants] - end - elsif RUBY_VERSION.match(/1\.8/) && alternatives.length.zero? + if RUBY_VERSION.match(/1\.8/) && alternatives.length.zero? puts 'WARNING: You should always pass the control alternative through as the second argument with any other alternatives as the third because the order of the hash is not preserved in ruby 1.8' end - ret = if Split.configuration.enabled - experiment_variable(alternatives, control, experiment_name) - else - control_variable(control) - end + begin + ret = if Split.configuration.enabled + load_and_start_trial(experiment_name, control, alternatives) + else + control_variable(control) + end + + rescue => e + raise(e) unless Split.configuration.db_failover + Split.configuration.db_failover_on_db_error.call(e) + + if Split.configuration.db_failover_allow_parameter_override && override_present?(experiment_name) + ret = override_alternative(experiment_name) + end + ensure + if ret.nil? + ret = control_variable(control) + end + end if block_given? if defined?(capture) # a block in a rails view @@ -29,37 +39,34 @@ def ab_test(experiment_name, control=nil, *alternatives) end end - def finished(experiment_name, options = {:reset => true}) - return if exclude_visitor? or !Split.configuration.enabled + def reset!(experiment) + ab_user.delete(experiment.key) + end - experiment = Split::Experiment.find(experiment_name) - if experiment.nil? - return unless Split.configuration.experiments - Split.configuration.experiments.each do |name,exp| - if exp[:metric] == experiment_name - local_opts = {} - local_opts[:reset] = exp[:resettable] unless exp[:resettable].nil? - finished name, options.merge(local_opts) - end + def finish_experiment(experiment, options = {:reset => true}) + should_reset = experiment.resettable? && options[:reset] + if ab_user[experiment.finished_key] && !should_reset + return true + else + alternative_name = ab_user[experiment.key] + trial = Trial.new(:experiment => experiment, :alternative_name => alternative_name) + trial.complete! + if should_reset + reset!(experiment) + else + ab_user[experiment.finished_key] = true end - return - end - - if Split.configuration.experiments && Split.configuration.experiments[experiment_name] - reset = Split.configuration.experiments[experiment_name][:resettable] - options[:reset] = reset unless reset.nil? end + end - return if !options[:reset] && ab_user[experiment.finished_key] - if alternative_name = ab_user[experiment.key] - alternative = Split::Alternative.new(alternative_name, experiment_name) - alternative.increment_completion + def finished(metric_name, options = {:reset => true}) + return if exclude_visitor? || Split.configuration.disabled? + experiments = Metric.possible_experiments(metric_name) - if options[:reset] - ab_user.delete(experiment.key) - else - ab_user[experiment.finished_key] = true + if experiments.any? + experiments.each do |experiment| + finish_experiment(experiment, options) end end rescue => e @@ -67,13 +74,18 @@ def finished(experiment_name, options = {:reset => true}) Split.configuration.db_failover_on_db_error.call(e) end - def override(experiment_name, alternatives) - params[experiment_name] if defined?(params) && alternatives.include?(params[experiment_name]) + def override_present?(experiment_name) + defined?(params) && params[experiment_name] + end + + def override_alternative(experiment_name) + params[experiment_name] if override_present?(experiment_name) end def begin_experiment(experiment, alternative_name = nil) alternative_name ||= experiment.control.name ab_user[experiment.key] = alternative_name + alternative_name end def ab_user @@ -81,7 +93,7 @@ def ab_user end def exclude_visitor? - is_robot? or is_ignored_ip_address? + is_robot? || is_ignored_ip_address? end def not_allowed_to_test?(experiment_key) @@ -125,78 +137,41 @@ def control_variable(control) Hash === control ? control.keys.first : control end - def experiment_variable(alternatives, control, experiment_name) - begin + def load_and_start_trial(experiment_name, control, alternatives) + if control.nil? && alternatives.length.zero? + experiment = Experiment.find(experiment_name) + + raise ExperimentNotFound("#{experiment_name} not found") if experiment.nil? + else experiment = Split::Experiment.find_or_create(experiment_name, *([control] + alternatives)) - if experiment.winner - ret = experiment.winner.name + end + + start_trial( Trial.new(:experiment => experiment) ) + end + + def start_trial(trial) + experiment = trial.experiment + if override_present?(experiment.name) + ret = override_alternative(experiment.name) + else + clean_old_versions(experiment) + if exclude_visitor? || not_allowed_to_test?(experiment.key) + ret = experiment.control.name else - if forced_alternative = override(experiment.name, experiment.alternative_names) - ret = forced_alternative + if ab_user[experiment.key] + ret = ab_user[experiment.key] else - clean_old_versions(experiment) - if exclude_visitor? or not_allowed_to_test?(experiment.key) - ret = experiment.control.name - else - if ab_user[experiment.key] - ret = ab_user[experiment.key] - else - alternative = experiment.next_alternative - alternative.increment_participation - begin_experiment(experiment, alternative.name) - ret = alternative.name - end - end + trial.choose! + ret = begin_experiment(experiment, trial.alternative.name) end end - rescue => e - raise unless Split.configuration.db_failover - Split.configuration.db_failover_on_db_error.call(e) - if Split.configuration.db_failover_allow_parameter_override - all_alternatives = *([control] + alternatives) - alternative_names = all_alternatives.map{|a| a.is_a?(Hash) ? a.keys : a}.flatten - ret = override(experiment_name, alternative_names) - end - unless ret - ret = control_variable(control) - end end + ret end def keys_without_experiment(keys, experiment_key) keys.reject { |k| k.match(Regexp.new("^#{experiment_key}(:finished)?$")) } end - - def normalize_variants(variants) - given_probability, num_with_probability = variants.inject([0,0]) do |a,v| - p, n = a - if v.kind_of?(Hash) && v[:percent] - [p + v[:percent], n + 1] - else - a - end - end - - num_without_probability = variants.length - num_with_probability - unassigned_probability = ((100.0 - given_probability) / num_without_probability / 100.0) - - if num_with_probability.nonzero? - variants = variants.map do |v| - if v.kind_of?(Hash) && v[:name] && v[:percent] - { v[:name] => v[:percent] / 100.0 } - elsif v.kind_of?(Hash) && v[:name] - { v[:name] => unassigned_probability } - else - { v => unassigned_probability } - end - end - [variants.shift, variants] - else - variants = variants.dup - [variants.shift, variants] - end - end end - end diff --git a/lib/split/metric.rb b/lib/split/metric.rb new file mode 100644 index 00000000..5841a60c --- /dev/null +++ b/lib/split/metric.rb @@ -0,0 +1,68 @@ +module Split + class Metric + attr_accessor :name + attr_accessor :experiments + + def initialize(attrs = {}) + attrs.each do |key,value| + if self.respond_to?("#{key}=") + self.send("#{key}=", value) + end + end + end + + def self.load_from_redis(name) + metric = Split.redis.hget(:metrics, name) + if metric + experiment_names = metric.split(',') + + experiments = experiment_names.collect do |experiment_name| + Split::Experiment.find(experiment_name) + end + + Split::Metric.new(:name => name, :experiments => experiments) + else + nil + end + end + + def self.load_from_configuration(name) + metrics = Split.configuration.metrics + if metrics && metrics[name] + Split::Metric.new(:experiments => metrics[name], :name => name) + else + nil + end + end + + def self.find(name) + name = name.intern if name.is_a?(String) + metric = load_from_configuration(name) + metric = load_from_redis(name) if metric.nil? + metric + end + + def self.possible_experiments(metric_name) + experiments = [] + metric = Split::Metric.find(metric_name) + if metric + experiments << metric.experiments + end + experiment = Split::Experiment.find(metric_name) + if experiment + experiments << experiment + end + experiments.flatten + end + + def save + Split.redis.hset(:metrics, name, experiments.map(&:name).join(',')) + end + + def complete! + experiments.each do |experiment| + experiment.complete! + end + end + end +end \ No newline at end of file diff --git a/lib/split/trial.rb b/lib/split/trial.rb new file mode 100644 index 00000000..b5f1b68b --- /dev/null +++ b/lib/split/trial.rb @@ -0,0 +1,43 @@ +module Split + class Trial + attr_accessor :experiment + attr_writer :alternative + + def initialize(attrs = {}) + self.experiment = attrs[:experiment] if !attrs[:experiment].nil? + self.alternative = attrs[:alternative] if !attrs[:alternative].nil? + self.alternative_name = attrs[:alternative_name] if !attrs[:alternative_name].nil? + end + + def alternative + @alternative ||= if experiment.winner + experiment.winner + end + end + + def complete! + alternative.increment_completion if alternative + end + + def choose! + choose + record! + end + + def record! + alternative.increment_participation + end + + def choose + if experiment.winner + self.alternative = experiment.winner + else + self.alternative = experiment.next_alternative + end + end + + def alternative_name=(name) + self.alternative= self.experiment.alternatives.find{|a| a.name == name } + end + end +end \ No newline at end of file diff --git a/spec/algorithms/weighted_sample_spec.rb b/spec/algorithms/weighted_sample_spec.rb new file mode 100644 index 00000000..5f3fc64c --- /dev/null +++ b/spec/algorithms/weighted_sample_spec.rb @@ -0,0 +1,18 @@ +require "spec_helper" + +describe Split::Algorithms::WeightedSample do + it "should return an alternative" do + experiment = Split::Experiment.find_or_create('link_color', {'blue' => 100}, {'red' => 0 }) + Split::Algorithms::WeightedSample.choose_alternative(experiment).class.should == Split::Alternative + end + + it "should always return a heavily weighted option" do + experiment = Split::Experiment.find_or_create('link_color', {'blue' => 100}, {'red' => 0 }) + Split::Algorithms::WeightedSample.choose_alternative(experiment).name.should == 'blue' + end + + it "should return one of the results" do + experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 }) + ['red', 'blue'].should include Split::Algorithms::WeightedSample.choose_alternative(experiment).name + end +end \ No newline at end of file diff --git a/spec/algorithms/whiplash_spec.rb b/spec/algorithms/whiplash_spec.rb new file mode 100644 index 00000000..09cb0d5c --- /dev/null +++ b/spec/algorithms/whiplash_spec.rb @@ -0,0 +1,23 @@ +require "spec_helper" + +describe Split::Algorithms::Whiplash do + + it "should return an algorithm" do + experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 }) + Split::Algorithms::Whiplash.choose_alternative(experiment).class.should == Split::Alternative + end + + it "should return one of the results" do + experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 }) + ['red', 'blue'].should include Split::Algorithms::Whiplash.choose_alternative(experiment).name + end + + it "should guess floats" do + Split::Algorithms::Whiplash.send(:arm_guess, 0, 0).class.should == Float + Split::Algorithms::Whiplash.send(:arm_guess, 1, 0).class.should == Float + Split::Algorithms::Whiplash.send(:arm_guess, 2, 1).class.should == Float + Split::Algorithms::Whiplash.send(:arm_guess, 1000, 5).class.should == Float + Split::Algorithms::Whiplash.send(:arm_guess, 10, -2).class.should == Float + end + +end \ No newline at end of file diff --git a/spec/alternative_spec.rb b/spec/alternative_spec.rb index fc228650..8ac4d2d9 100644 --- a/spec/alternative_spec.rb +++ b/spec/alternative_spec.rb @@ -4,16 +4,79 @@ describe Split::Alternative do it "should have a name" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) alternative = Split::Alternative.new('Basket', 'basket_text') alternative.name.should eql('Basket') end it "return only the name" do - experiment = Split::Experiment.new('basket_text', {'Basket' => 0.6}, {"Cart" => 0.4}) + experiment = Split::Experiment.new('basket_text', :alternative_names => [{'Basket' => 0.6}, {"Cart" => 0.4}]) alternative = Split::Alternative.new('Basket', 'basket_text') alternative.name.should eql('Basket') end + + describe 'weights' do + + it "should set the weights" do + experiment = Split::Experiment.new('basket_text', :alternative_names => [{'Basket' => 0.6}, {"Cart" => 0.4}]) + first = experiment.alternatives[0] + first.name.should == 'Basket' + first.weight.should == 0.6 + + second = experiment.alternatives[1] + second.name.should == 'Cart' + second.weight.should == 0.4 + end + + it "accepts probability on variants" do + Split.configuration.experiments = { + :my_experiment => { + :variants => [ + { :name => "control_opt", :percent => 67 }, + { :name => "second_opt", :percent => 10 }, + { :name => "third_opt", :percent => 23 }, + ], + } + } + experiment = Split::Experiment.find(:my_experiment) + first = experiment.alternatives[0] + first.name.should == 'control_opt' + first.weight.should == 0.67 + + second = experiment.alternatives[1] + second.name.should == 'second_opt' + second.weight.should == 0.1 + end + + # it "accepts probability on some variants" do + # Split.configuration.experiments[:my_experiment] = { + # :variants => [ + # { :name => "control_opt", :percent => 34 }, + # "second_opt", + # { :name => "third_opt", :percent => 23 }, + # "fourth_opt", + # ], + # } + # should start_experiment(:my_experiment).with({"control_opt" => 0.34}, {"second_opt" => 0.215}, {"third_opt" => 0.23}, {"fourth_opt" => 0.215}) + # ab_test :my_experiment + # end + # + # it "allows name param without probability" do + # Split.configuration.experiments[:my_experiment] = { + # :variants => [ + # { :name => "control_opt" }, + # "second_opt", + # { :name => "third_opt", :percent => 64 }, + # ], + # } + # should start_experiment(:my_experiment).with({"control_opt" => 0.18}, {"second_opt" => 0.18}, {"third_opt" => 0.64}) + # ab_test :my_experiment + # end + + it "should set the weights from a configuration file" do + + end + end it "should have a default participation count of 0" do alternative = Split::Alternative.new('Basket', 'basket_text') @@ -26,7 +89,7 @@ end it "should belong to an experiment" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) experiment.save alternative = Split::Alternative.new('Basket', 'basket_text') alternative.experiment.name.should eql(experiment.name) @@ -39,7 +102,7 @@ end it "should increment participation count" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) experiment.save alternative = Split::Alternative.new('Basket', 'basket_text') old_participant_count = alternative.participant_count @@ -50,7 +113,7 @@ end it "should increment completed count" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) experiment.save alternative = Split::Alternative.new('Basket', 'basket_text') old_completed_count = alternative.participant_count @@ -70,7 +133,7 @@ end it "should know if it is the control of an experiment" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) experiment.save alternative = Split::Alternative.new('Basket', 'basket_text') alternative.control?.should be_true @@ -80,7 +143,7 @@ describe 'unfinished_count' do it "should be difference between participant and completed counts" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) experiment.save alternative = Split::Alternative.new('Basket', 'basket_text') alternative.increment_participation diff --git a/spec/configuration_spec.rb b/spec/configuration_spec.rb index 06c30c27..fbc2a8e6 100644 --- a/spec/configuration_spec.rb +++ b/spec/configuration_spec.rb @@ -21,6 +21,11 @@ @config.enabled.should be_true end + it "disabled is the opposite of enabled" do + @config.enabled = false + @config.disabled?.should be_true + end + it "should provide a default pattern for robots" do %w[Baidu Gigabot Googlebot libwww-perl lwp-trivial msnbot SiteUptime Slurp WordPress ZIBB ZyBorg].each do |robot| @config.robot_regex.should =~ robot @@ -30,4 +35,32 @@ it "should use the session adapter for persistence by default" do @config.persistence.should eq(Split::Persistence::SessionAdapter) end + + it "should load a metric" do + @config.experiments = {:my_experiment=> + {:variants=>["control_opt", "other_opt"], :metric=>:my_metric}} + + @config.metrics.should_not be_nil + @config.metrics.keys.should == [:my_metric] + end + + it "should allow loading of experiment using experment_for" do + @config.experiments = {:my_experiment=> + {:variants=>["control_opt", "other_opt"], :metric=>:my_metric}} + @config.experiment_for(:my_experiment).should == {:variants=>["control_opt", ["other_opt"]]} + end + + it "should normalize experiments" do + @config.experiments = { + :my_experiment => { + :variants => [ + { :name => "control_opt", :percent => 67 }, + { :name => "second_opt", :percent => 10 }, + { :name => "third_opt", :percent => 23 }, + ], + } + } + + @config.normalized_experiments.should == {:my_experiment=>{:variants=>[{"control_opt"=>0.67}, [{"second_opt"=>0.1}, {"third_opt"=>0.23}]]}} + end end \ No newline at end of file diff --git a/spec/experiment_spec.rb b/spec/experiment_spec.rb index 9431ce63..2ea72075 100644 --- a/spec/experiment_spec.rb +++ b/spec/experiment_spec.rb @@ -1,54 +1,135 @@ require 'spec_helper' require 'split/experiment' +require 'split/algorithms' +require 'time' describe Split::Experiment do - it "should have a name" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.name.should eql('basket_text') - end + context "with an experiment" do + let(:experiment) { Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) } + + it "should have a name" do + experiment.name.should eql('basket_text') + end - it "should have alternatives" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.alternatives.length.should be 2 - end + it "should have alternatives" do + experiment.alternatives.length.should be 2 + end + + it "should have alternatives with correct names" do + experiment.alternatives.collect{|a| a.name}.should == ['Basket', 'Cart'] + end + + it "should be resettable by default" do + experiment.resettable.should be_true + end + + it "should save to redis" do + experiment.save + Split.redis.exists('basket_text').should be true + end - it "should save to redis" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.save - Split.redis.exists('basket_text').should be true - end + it "should save the start time to redis" do + experiment_start_time = Time.parse("Sat Mar 03 14:01:03") + Time.stub(:now => experiment_start_time) + experiment.save - it "should save the start time to redis" do - experiment_start_time = Time.parse("Sat Mar 03 14:01:03") - Time.stub(:now => experiment_start_time) - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.save + Split::Experiment.find('basket_text').start_time.should == experiment_start_time + end + + it "should save the selected algorithm to redis" do + experiment_algorithm = Split::Algorithms::Whiplash + experiment.algorithm = experiment_algorithm + experiment.save - Split::Experiment.find('basket_text').start_time.should == experiment_start_time - end + Split::Experiment.find('basket_text').algorithm.should == experiment_algorithm + end - it "should handle not having a start time" do - experiment_start_time = Time.parse("Sat Mar 03 14:01:03") - Time.stub(:now => experiment_start_time) - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.save + it "should handle not having a start time" do + experiment_start_time = Time.parse("Sat Mar 03 14:01:03") + Time.stub(:now => experiment_start_time) + experiment.save - Split.redis.hdel(:experiment_start_times, experiment.name) + Split.redis.hdel(:experiment_start_times, experiment.name) - Split::Experiment.find('basket_text').start_time.should == nil - end + Split::Experiment.find('basket_text').start_time.should == nil + end - it "should not create duplicates when saving multiple times" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.save - experiment.save - Split.redis.exists('basket_text').should be true - Split.redis.lrange('basket_text', 0, -1).should eql(['Basket', "Cart"]) + it "should not create duplicates when saving multiple times" do + experiment.save + experiment.save + Split.redis.exists('basket_text').should be true + Split.redis.lrange('basket_text', 0, -1).should eql(['Basket', "Cart"]) + end + + describe 'new record?' do + it "should know if it hasn't been saved yet" do + experiment.new_record?.should be_true + end + + it "should know if it has been saved yet" do + experiment.save + experiment.new_record?.should be_false + end + end + + describe 'find' do + it "should return an existing experiment" do + experiment.save + Split::Experiment.find('basket_text').name.should eql('basket_text') + end + + it "should return an existing experiment" do + Split::Experiment.find('non_existent_experiment').should be_nil + end + end + + describe 'control' do + it 'should be the first alternative' do + experiment.save + experiment.control.name.should eql('Basket') + end + end + end + describe 'initialization' do + it "should set the algorithm when passed as an option to the initializer" do + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash) + experiment.algorithm.should == Split::Algorithms::Whiplash + end + + it "should be possible to make an experiment not resettable" do + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :resettable => false) + experiment.resettable.should be_false + end + end + + describe 'persistent configuration' do + + it "should persist resettable in redis" do + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :resettable => false) + experiment.save + + e = Split::Experiment.find('basket_text') + e.should == experiment + e.resettable.should be_false + + end + + it "should persist algorithm in redis" do + experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash) + experiment.save + + e = Split::Experiment.find('basket_text') + e.should == experiment + e.algorithm.should == Split::Algorithms::Whiplash + end end + + + describe 'deleting' do it 'should delete itself' do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") + experiment = Split::Experiment.new('basket_text', :alternative_names => [ 'Basket', "Cart"]) experiment.save experiment.delete @@ -64,39 +145,6 @@ end end - describe 'new record?' do - it "should know if it hasn't been saved yet" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.new_record?.should be_true - end - - it "should know if it has been saved yet" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.save - experiment.new_record?.should be_false - end - end - - describe 'find' do - it "should return an existing experiment" do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.save - Split::Experiment.find('basket_text').name.should eql('basket_text') - end - - it "should return an existing experiment" do - Split::Experiment.find('non_existent_experiment').should be_nil - end - end - - describe 'control' do - it 'should be the first alternative' do - experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") - experiment.save - experiment.control.name.should eql('Basket') - end - end - describe 'winner' do it "should have no winner initially" do experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') @@ -148,10 +196,24 @@ experiment.version.should eql(1) end end - + + describe 'algorithm' do + let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') } + + it 'should use the default algorithm if none is specified' do + experiment.algorithm.should == Split.configuration.algorithm + end + + it 'should use the user specified algorithm for this experiment if specified' do + experiment.algorithm = Split::Algorithms::Whiplash + experiment.algorithm.should == Split::Algorithms::Whiplash + end + end + describe 'next_alternative' do + let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') } + it "should always return the winner if one exists" do - experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') green = Split::Alternative.new('green', 'link_color') experiment.winner = 'green' @@ -161,6 +223,19 @@ experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') experiment.next_alternative.name.should eql('green') end + + it "should use the specified algorithm if a winner does not exist" do + Split.configuration.algorithm.should_receive(:choose_alternative).and_return(Split::Alternative.new('green', 'link_color')) + experiment.next_alternative.name.should eql('green') + end + end + + describe 'single alternative' do + let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue') } + + it "should always return the color blue" do + experiment.next_alternative.name.should eql('blue') + end end describe 'changing an existing experiment' do @@ -207,7 +282,4 @@ same_experiment.alternatives.map(&:weight).should == [1, 2] end end - - - end diff --git a/spec/helper_spec.rb b/spec/helper_spec.rb index 0471166d..8c0d8165 100755 --- a/spec/helper_spec.rb +++ b/spec/helper_spec.rb @@ -81,6 +81,8 @@ ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2) experiment = Split::Experiment.find('link_color') experiment.alternative_names.should eql(['blue', 'red']) + # TODO: persist alternative weights + # experiment.alternatives.collect{|a| a.weight}.should == [0.01, 0.2] end it "should only let a user participate in one experiment at a time" do @@ -197,7 +199,8 @@ def should_finish_experiment(experiment_name, should_finish=true) alts = Split.configuration.experiments[experiment_name][:variants] experiment = Split::Experiment.find_or_create(experiment_name, *alts) alt_name = ab_user[experiment.key] = alts.first - alt = double + alt = mock('alternative') + alt.stub(:name).and_return(alt_name) Split::Alternative.stub(:new).with(alt_name, experiment_name).and_return(alt) if should_finish alt.should_receive(:increment_completion) @@ -569,57 +572,39 @@ def should_finish_experiment(experiment_name, should_finish=true) finished('link_color') end end - - end - end context "with preloaded config" do - before { Split.configuration.experiments = {} } - - subject { self } - RSpec::Matchers.define :start_experiment do |name| - match do |actual| - @control ||= anything - @alternatives ||= anything - @times ||= 1 - actual.should_receive(:experiment_variable).with(@alternatives, @control, name).exactly(@times).times - end - chain :with do |control, *alternatives| - @control = control - @alternatives = alternatives.flatten - end - chain :exactly do |times| - @times = times - end - chain :times do end - end - + before { Split.configuration.experiments = {}} + it "pulls options from config file" do Split.configuration.experiments[:my_experiment] = { :variants => [ "control_opt", "other_opt" ], } - should start_experiment(:my_experiment).with("control_opt", ["other_opt"]) ab_test :my_experiment + Split::Experiment.find(:my_experiment).alternative_names.should == [ "control_opt", "other_opt" ] end - + it "can be called multiple times" do Split.configuration.experiments[:my_experiment] = { :variants => [ "control_opt", "other_opt" ], } - should start_experiment(:my_experiment).with("control_opt", ["other_opt"]).exactly(5).times 5.times { ab_test :my_experiment } + experiment = Split::Experiment.find(:my_experiment) + experiment.alternative_names.should == [ "control_opt", "other_opt" ] + experiment.participant_count.should == 1 end - + it "accepts multiple variants" do Split.configuration.experiments[:my_experiment] = { :variants => [ "control_opt", "second_opt", "third_opt" ], } - should start_experiment(:my_experiment).with("control_opt", ["second_opt", "third_opt"]) ab_test :my_experiment + experiment = Split::Experiment.find(:my_experiment) + experiment.alternative_names.should == [ "control_opt", "second_opt", "third_opt" ] end - + it "accepts probability on variants" do Split.configuration.experiments[:my_experiment] = { :variants => [ @@ -628,10 +613,12 @@ def should_finish_experiment(experiment_name, should_finish=true) { :name => "third_opt", :percent => 23 }, ], } - should start_experiment(:my_experiment).with({"control_opt" => 0.67}, {"second_opt" => 0.1}, {"third_opt" => 0.23}) ab_test :my_experiment + experiment = Split::Experiment.find(:my_experiment) + experiment.alternatives.collect{|a| [a.name, a.weight]}.should == [['control_opt', 0.67], ['second_opt', 0.1], ['third_opt', 0.23]] + end - + it "accepts probability on some variants" do Split.configuration.experiments[:my_experiment] = { :variants => [ @@ -641,10 +628,13 @@ def should_finish_experiment(experiment_name, should_finish=true) "fourth_opt", ], } - should start_experiment(:my_experiment).with({"control_opt" => 0.34}, {"second_opt" => 0.215}, {"third_opt" => 0.23}, {"fourth_opt" => 0.215}) ab_test :my_experiment + experiment = Split::Experiment.find(:my_experiment) + names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]} + names_and_weights.should == [['control_opt', 0.34], ['second_opt', 0.215], ['third_opt', 0.23], ['fourth_opt', 0.215]] + names_and_weights.inject(0){|sum, nw| sum + nw[1]}.should == 1.0 end - + it "allows name param without probability" do Split.configuration.experiments[:my_experiment] = { :variants => [ @@ -653,8 +643,11 @@ def should_finish_experiment(experiment_name, should_finish=true) { :name => "third_opt", :percent => 64 }, ], } - should start_experiment(:my_experiment).with({"control_opt" => 0.18}, {"second_opt" => 0.18}, {"third_opt" => 0.64}) ab_test :my_experiment + experiment = Split::Experiment.find(:my_experiment) + names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]} + names_and_weights.should == [['control_opt', 0.18], ['second_opt', 0.18], ['third_opt', 0.64]] + names_and_weights.inject(0){|sum, nw| sum + nw[1]}.should == 1.0 end end diff --git a/spec/metric_spec.rb b/spec/metric_spec.rb new file mode 100644 index 00000000..dcbff4e0 --- /dev/null +++ b/spec/metric_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' +require 'split/metric' + +describe Split::Metric do + describe 'possible experiments' do + it "should load the experiment if there is one, but no metric" do + experiment = Split::Experiment.find_or_create('color', 'red', 'blue') + Split::Metric.possible_experiments('color').should == [experiment] + end + + it "should load the experiments in a metric" do + experiment1 = Split::Experiment.find_or_create('color', 'red', 'blue') + experiment2 = Split::Experiment.find_or_create('size', 'big', 'small') + + metric = Split::Metric.new(:name => 'purchase', :experiments => [experiment1, experiment2]) + metric.save + Split::Metric.possible_experiments('purchase').should include(experiment1, experiment2) + end + + it "should load both the metric experiments and an experiment with the same name" do + experiment1 = Split::Experiment.find_or_create('purchase', 'red', 'blue') + experiment2 = Split::Experiment.find_or_create('size', 'big', 'small') + + metric = Split::Metric.new(:name => 'purchase', :experiments => [experiment2]) + metric.save + Split::Metric.possible_experiments('purchase').should include(experiment1, experiment2) + end + end + +end diff --git a/spec/trial_spec.rb b/spec/trial_spec.rb new file mode 100644 index 00000000..2fad43cb --- /dev/null +++ b/spec/trial_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' +require 'split/trial' + +describe Split::Trial do + it "should be initializeable" do + experiment = mock('experiment') + alternative = mock('alternative') + trial = Split::Trial.new(:experiment => experiment, :alternative => alternative) + trial.experiment.should == experiment + trial.alternative.should == alternative + end + + describe "alternative" do + it "should use the alternative if specified" do + trial = Split::Trial.new(:experiment => experiment = mock('experiment'), :alternative => alternative = mock('alternative')) + trial.should_not_receive(:choose) + trial.alternative.should == alternative + end + + it "should populate alternative with a full alternative object after calling choose" do + experiment = Split::Experiment.new('basket_text', :alternative_names => ['basket', 'cart']) + experiment.save + trial = Split::Trial.new(:experiment => experiment) + trial.choose + trial.alternative.class.should == Split::Alternative + ['basket', 'cart'].should include(trial.alternative.name) + end + + it "should populate an alternative when only one option is offerred" do + experiment = Split::Experiment.new('basket_text', :alternative_names => ['basket']) + experiment.save + trial = Split::Trial.new(:experiment => experiment) + trial.choose + trial.alternative.class.should == Split::Alternative + trial.alternative.name.should == 'basket' + end + + + it "should choose from the available alternatives" do + trial = Split::Trial.new(:experiment => experiment = mock('experiment')) + experiment.should_receive(:next_alternative).and_return(alternative = mock('alternative')) + alternative.should_receive(:increment_participation) + experiment.should_receive(:winner).and_return nil + trial.choose! + + trial.alternative.should == alternative + end + end + + describe "alternative_name" do + it "should load the alternative when the alternative name is set" do + experiment = Split::Experiment.new('basket_text', :alternative_names => ['basket', "cart"]) + experiment.save + + trial = Split::Trial.new(:experiment => experiment, :alternative_name => 'basket') + trial.alternative.name.should == 'basket' + end + end +end diff --git a/split.gemspec b/split.gemspec index fe473679..01360ccb 100644 --- a/split.gemspec +++ b/split.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |s| s.add_dependency 'redis', '>= 2.1' s.add_dependency 'redis-namespace', '>= 1.1.0' s.add_dependency 'sinatra', '>= 1.2.6' + s.add_dependency 'simple-random' # Ruby 1.8 doesn't include JSON in the std lib if RUBY_VERSION < "1.9"