Skip to content

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also .

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also .
...
  • 6 commits
  • 8 files changed
  • 0 commit comments
  • 1 contributor
View
16 generators/templates/vanity_migration.rb
@@ -23,24 +23,24 @@ def self.up
add_index :vanity_experiments, [:experiment_id]
create_table :vanity_conversions do |t|
- t.integer :vanity_experiment_id
+ t.references :vanity_experiment
t.integer :alternative
t.integer :conversions
end
add_index :vanity_conversions, [:vanity_experiment_id, :alternative], :name => "by_experiment_id_and_alternative"
create_table :vanity_participants do |t|
- t.string :experiment_id
- t.string :identity
+ t.references :vanity_experiment
+ t.integer :identity
t.integer :shown
t.integer :seen
t.integer :converted
+ t.timestamps
end
- add_index :vanity_participants, [:experiment_id]
- add_index :vanity_participants, [:experiment_id, :identity], :name => "by_experiment_id_and_identity"
- add_index :vanity_participants, [:experiment_id, :shown], :name => "by_experiment_id_and_shown"
- add_index :vanity_participants, [:experiment_id, :seen], :name => "by_experiment_id_and_seen"
- add_index :vanity_participants, [:experiment_id, :converted], :name => "by_experiment_id_and_converted"
+ add_index :vanity_participants, [:vanity_experiment_id, :identity], :name => "by_experiment_id_and_identity"
+ add_index :vanity_participants, [:vanity_experiment_id, :shown], :name => "by_experiment_id_and_shown"
+ add_index :vanity_participants, [:vanity_experiment_id, :seen], :name => "by_experiment_id_and_seen"
+ add_index :vanity_participants, [:vanity_experiment_id, :converted], :name => "by_experiment_id_and_converted"
end
def self.down
View
16 lib/generators/templates/vanity_migration.rb
@@ -23,24 +23,24 @@ def self.up
add_index :vanity_experiments, [:experiment_id]
create_table :vanity_conversions do |t|
- t.integer :vanity_experiment_id
+ t.references :vanity_experiment
t.integer :alternative
t.integer :conversions
end
add_index :vanity_conversions, [:vanity_experiment_id, :alternative], :name => "by_experiment_id_and_alternative"
create_table :vanity_participants do |t|
- t.string :experiment_id
- t.string :identity
+ t.references :vanity_experiment
+ t.integer :identity
t.integer :shown
t.integer :seen
t.integer :converted
+ t.timestamps
end
- add_index :vanity_participants, [:experiment_id]
- add_index :vanity_participants, [:experiment_id, :identity], :name => "by_experiment_id_and_identity"
- add_index :vanity_participants, [:experiment_id, :shown], :name => "by_experiment_id_and_shown"
- add_index :vanity_participants, [:experiment_id, :seen], :name => "by_experiment_id_and_seen"
- add_index :vanity_participants, [:experiment_id, :converted], :name => "by_experiment_id_and_converted"
+ add_index :vanity_participants, [:vanity_experiment_id, :identity], :name => "by_experiment_id_and_identity"
+ add_index :vanity_participants, [:vanity_experiment_id, :shown], :name => "by_experiment_id_and_shown"
+ add_index :vanity_participants, [:vanity_experiment_id, :seen], :name => "by_experiment_id_and_seen"
+ add_index :vanity_participants, [:vanity_experiment_id, :converted], :name => "by_experiment_id_and_converted"
end
def self.down
View
5 lib/vanity/adapters/abstract_adapter.rb
@@ -110,6 +110,11 @@ def ab_add_participant(experiment, alternative, identity)
fail "Not implemented"
end
+ # Indicates which alternative has been picked for this participant. See #ab_add_participant.
+ def ab_chosen(experiment, identity)
+ false # TODO: default to false for now; should add to each adapter
+ end
+
# Records a conversion in this experiment for the given alternative.
# Associates a value with the conversion (default to 1). If implicit is
# true, add particpant if not already recorded for this experiment. If
View
25 lib/vanity/adapters/active_record_adapter.rb
@@ -39,6 +39,7 @@ class VanityMetricValue < VanityRecord
class VanityExperiment < VanityRecord
set_table_name :vanity_experiments
has_many :vanity_conversions, :dependent => :destroy
+ has_many :vanity_participants
# Finds or creates the experiment
def self.retrieve(experiment)
@@ -60,17 +61,20 @@ class VanityConversion < VanityRecord
# Participant model
class VanityParticipant < VanityRecord
set_table_name :vanity_participants
+ belongs_to :vanity_experiment
# Finds the participant by experiment and identity. If
# create is true then it will create the participant
# if not found. If a hash is passed then this will be
# passed to create if creating, or will be used to
# update the found participant.
def self.retrieve(experiment, identity, create = true, update_with = nil)
- if record = VanityParticipant.first(:conditions=>{ :experiment_id=>experiment.to_s, :identity=>identity.to_s })
+ exp = VanityExperiment.retrieve(experiment)
+ raise "no experiment found #{experiment}" unless exp
+ if record = exp.vanity_participants.first(:conditions=>{ :identity=>identity })
record.update_attributes(update_with) if update_with
elsif create
- record = VanityParticipant.create({ :experiment_id=>experiment.to_s, :identity=>identity }.merge(update_with || {}))
+ record = VanityParticipant.create({ :vanity_experiment_id=>exp.id, :identity=>identity }.merge(update_with || {}))
end
record
end
@@ -177,8 +181,8 @@ def is_experiment_completed?(experiment)
# :conversions.
def ab_counts(experiment, alternative)
record = VanityExperiment.retrieve(experiment)
- participants = VanityParticipant.count(:conditions => {:experiment_id => experiment.to_s, :seen => alternative})
- converted = VanityParticipant.count(:conditions => {:experiment_id => experiment.to_s, :converted => alternative})
+ participants = record.vanity_participants.count(:conditions => {:seen => alternative})
+ converted = record.vanity_participants.count(:conditions => {:converted => alternative})
conversions = record.vanity_conversions.sum(:conversions, :conditions => {:alternative => alternative})
{
@@ -211,6 +215,12 @@ def ab_add_participant(experiment, alternative, identity)
VanityParticipant.retrieve(experiment, identity, true, :seen => alternative)
end
+ # Indicates which alternative has been picked for this participant. See #ab_add_participant.
+ def ab_chosen(experiment, identity)
+ participant = VanityParticipant.retrieve(experiment, identity, false)
+ participant && participant.seen
+ end
+
# Records a conversion in this experiment for the given alternative.
# Associates a value with the conversion (default to 1). If implicit is
# true, add particpant if not already recorded for this experiment. If
@@ -234,9 +244,10 @@ def ab_set_outcome(experiment, alternative = 0)
# Deletes all information about this experiment.
def destroy_experiment(experiment)
- VanityParticipant.delete_all(:experiment_id => experiment.to_s)
- record = VanityExperiment.find_by_experiment_id(experiment.to_s)
- record && record.destroy
+ if record = VanityExperiment.find_by_experiment_id(experiment.to_s)
+ record.vanity_participants.delete_all
+ record.destroy
+ end
end
def to_s
View
107 lib/vanity/experiment/ab_test.rb
@@ -6,10 +6,11 @@ module Experiment
# One of several alternatives in an A/B test (see AbTest#alternatives).
class Alternative
- def initialize(experiment, id, value) #, participants, converted, conversions)
+ def initialize(experiment, id, key, value) #, participants, converted, conversions)
@experiment = experiment
@id = id
@name = "option #{(@id + 65).chr}"
+ @key = key
@value = value
end
@@ -19,6 +20,9 @@ def initialize(experiment, id, value) #, participants, converted, conversions)
# Alternative name (option A, option B, etc).
attr_reader :name
+ # Alternative key.
+ attr_reader :key
+
# Alternative value.
attr_reader :value
@@ -76,7 +80,7 @@ def to_s
end
def inspect
- "#{name}: #{value} #{converted}/#{participants}"
+ "#{name}: #{key} #{converted}/#{participants}"
end
def load_counts
@@ -145,28 +149,48 @@ def metrics(*args)
# puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
def alternatives(*args)
@alternatives = args.empty? ? [true, false] : args.clone
+ if @control
+ @alternatives.delete(@control)
+ @alternatives.push(@control) # move to end
+ end
class << self
define_method :alternatives, instance_method(:_alternatives)
end
nil
end
+ # Call this method to add complex values for alternatives, and use
+ # alternative values as keys to access them.
+ #
+ # @example Define A/B test with complex values
+ # ab_test "Option type" do
+ # metrics :coolness
+ # alternatives "a", "b", "c"
+ # values "a" => {:some => :data},
+ # "b" => {:more => :data}, "c" => {:other => :data}
+ # end
+ def values(data)
+ raise ArgumentError, "values should respond to []" unless data.respond_to?(:[])
+ @values = data.clone
+ end
+
def _alternatives
alts = []
- @alternatives.each_with_index do |value, i|
- alts << Alternative.new(self, i, value)
+ @alternatives.each_with_index do |key, i|
+ value = (@values[key] if @values) || key
+ alts << Alternative.new(self, i, key, value)
end
alts
end
private :_alternatives
- # Returns an Alternative with the specified value.
+ # Returns an Alternative with the specified key.
#
# @example
# alternative(:red) == alternatives[0]
# alternative(:blue) == alternatives[2]
- def alternative(value)
- alternatives.find { |alt| alt.value == value }
+ def alternative(key)
+ alternatives.find { |alt| alt.key == key }
end
# Defines an A/B test with two alternatives: false and true. This is the
@@ -244,26 +268,26 @@ def fingerprint(alternative)
# teardown do
# experiment(:green_button).select(nil)
# end
- def chooses(value)
+ def chooses(key)
if @playground.collecting?
- if value.nil?
+ if key.nil?
connection.ab_not_showing @id, identity
else
- index = @alternatives.index(value)
+ index = @alternatives.index(key)
#add them to the experiment unless they are already in it
unless index == connection.ab_showing(@id, identity)
connection.ab_add_participant @id, index, identity
check_completion!
end
- raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
+ raise ArgumentError, "No alternative #{key.inspect} for #{name}" unless index
if (connection.ab_showing(@id, identity) && connection.ab_showing(@id, identity) != index) ||
alternative_for(identity) != index
connection.ab_show @id, identity, index
end
end
else
@showing ||= {}
- @showing[identity] = value.nil? ? nil : @alternatives.index(value)
+ @showing[identity] = key.nil? ? nil : @alternatives.index(key)
end
self
end
@@ -279,6 +303,44 @@ def showing?(alternative)
end
end
+ # True if this experiment has been selected for the current id (see #chooses).
+ def chosen?
+ # True if experiment is active and a value has been chosen for current identity
+ !!if @playground.collecting? # return a boolean value
+ active? && (connection.ab_showing(@id, identity()) ||
+ connection.ab_chosen(@id, identity()))
+ # TODO: implement ab_chosen on all vanity adapters!!
+ else
+ @showing && @showing[identity()]
+ end
+ end
+
+ def assign_on(event)
+ @assign_event = event
+ end
+
+ def assign_on?(event)
+ @assign_event == event
+ end
+
+ def test_percent(pct)
+ raise RuntimeError, "Test percent must be an integer" unless pct.kind_of?(Integer)
+ @test_pct = pct
+ end
+
+ def control_percent(pct)
+ raise RuntimeError, "Control percent must be an integer" unless pct.kind_of?(Integer)
+ test_percent(100-pct)
+ end
+
+ def control_value(value)
+ @control = value
+ if @alternatives
+ @alternatives.delete(@control)
+ @alternatives.push(@control) # move to end
+ end
+ end
+
# -- Reporting --
@@ -463,12 +525,12 @@ def fake(values)
participants.times do |identity|
index = @alternatives.index(value)
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
- connection.ab_add_participant @id, index, "#{index}:#{identity}"
+ connection.ab_add_participant @id, index, index*10000+identity #"#{index}:#{identity}"
end
conversions.times do |identity|
index = @alternatives.index(value)
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
- connection.ab_add_conversion @id, index, "#{index}:#{identity}"
+ connection.ab_add_conversion @id, index, index*10000+identity #"#{index}:#{identity}"
end
end
end
@@ -478,9 +540,24 @@ def fake(values)
# identity, and randomly distributed alternatives for each identity (in the
# same experiment).
def alternative_for(identity)
- Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
+ id_hash = Digest::MD5.hexdigest("#{name}/#{identity}").to_i(16)
+ return hash_to_alternative(id_hash)
end
+ def hash_to_alternative(id_hash)
+ alternatives_count = @alternatives.size
+ if @test_pct
+ if (id_hash % 100 >= @test_pct)
+ return alternatives_count-1 #@control
+ else
+ id_hash = id_hash / 100
+ alternatives_count -= 1
+ end
+ end
+ return id_hash % alternatives_count
+ end
+
+
begin
a = 50.0
# Returns array of [z-score, percentage]
View
5 lib/vanity/helpers.rb
@@ -50,6 +50,11 @@ def ab_test(name, &block)
end
end
+ # Check whether the specified experiment has had a value chosen yet
+ def ab_test_chosen?(id)
+ Vanity.playground.experiment(id).chosen?
+ end
+
# Tracks an action associated with a metric.
#
# @example
View
11 lib/vanity/playground.rb
@@ -215,6 +215,17 @@ def track!(id, count = 1)
metric(id).track! count
end
+ # Choose a value for any experiment tagged with the specified event;
+ # filter can be used to specify a white-list for experiments.
+ def assign_on(event, filter=nil)
+ experiments.each do |id, obj|
+ if !filter || filter.include?(id)
+ if obj.assign_on?(event)
+ obj.choose
+ end
+ end
+ end
+ end
# -- Connection management --
View
2 vanity.gemspec → lookout-vanity.gemspec
@@ -2,7 +2,7 @@ $: << File.dirname(__FILE__) + "/lib"
require "vanity/version"
Gem::Specification.new do |spec|
- spec.name = "vanity"
+ spec.name = "lookout-vanity"
spec.version = Vanity::VERSION
spec.author = "Assaf Arkin"
spec.email = "assaf@labnotes.org"

No commit comments for this range

Something went wrong with that request. Please try again.