test/experiment_test.rb @@ -1,27 +1,35 @@ require "rake/rdoctask" +require "rake/testtask" spec = Gem::Specification.load(File.expand_path("vanity.gemspec", File.dirname(__FILE__))) +desc "Push new release to gemcutter and git tag" task :push do - version = spec.version.to_s.freeze sh "git push" - puts "Tagging version #{version} .." - sh "git tag #{version}" + puts "Tagging version #{spec.version} .." + sh "git tag #{spec.version}" sh "git push --tag" puts "Building and pushing gem .." - sh "gem build vanity.gemspec" - sh "gem push vanity-#{version}.gem" + sh "gem build #{spec.name}.gemspec" + sh "gem push #{spec.name}-#{spec.version}.gem" +end + +desc "Install #{spec.name} locally" +task :install do + sh "gem build #{spec.name}.gemspec" + sudo = "sudo" unless File.writable?( Gem::ConfigMap[:bindir]) + sh "#{sudo} gem install #{spec.name}-#{spec.version}.gem" end task :default=>:test -task :test do - FileList["test/**_test.rb"].each do |file| - require file - end +desc "Run all tests" +Rake::TestTask.new do |task| + task.test_files = FileList['test/*_test.rb'] + task.verbose = true + #task.warning = true end Rake::RDocTask.new(:rdoc) do |rdoc| rdoc.rdoc_files.include "README.rdoc", "lib/**/*.rb" rdoc.options = spec.rdoc_options - rdoc.title = "Vanity #{spec.version}" end Rakefile @@ -3,6 +3,11 @@ module Vanity # The meat. class AbTest < Base + def initialize(*args) #:nodoc: + super + @alternatives = [true, false] + end + # Chooses a value for the given identity. This method returns different # values for different identities, with random distribution, and consistently # returns the same value for the same identity. @@ -28,11 +33,8 @@ module Vanity # alternatives "red", "blue", "orange" # end def alternatives(*args) - if @alternatives - fail "Cannot change alternatives between runs" unless @alternatives == args - else - @alternatives = args - end + @alternatives = args unless args.empty? + @alternatives end # True/false A/B test. For example: lib/vanity/experiment/ab_test.rb @@ -6,13 +6,24 @@ module Vanity def initialize(name, &block) @name = name.to_s @namespace = "#{Vanity.playground.namespace}:experiments:#{name.downcase.gsub(/\W/, "_")}" + created = redis.get(key(:created_at)) || (redis.setnx(key(:created_at), Time.now.to_i) ; redis.get(key(:created_at))) + @created_at = Time.at(created.to_i) end - attr_reader :name + attr_reader :name, :created_at + + # Sets of returs description. For example + # experiment :simple do + # description "Simple experiment" + # end + # puts "Just defined: " + experiment(:simple).description + def description(text = nil) + @description = text if text + @description + end # Called to save the experiment definition. def save #:nodoc: - redis.set key, to_yaml end protected lib/vanity/experiment/base.rb @@ -2,22 +2,19 @@ require "test/test_helper" class AbTestTest < MiniTest::Spec it "uses A/B test when type: :ab_test" do - experiment :ab, type: :ab_test do - true_false - end + experiment(:ab, type: :ab_test) { } assert_instance_of Vanity::Experiment::AbTest, experiment(:ab) end it "uses A/B as default test type" do - experiment :default do - true_false - end + experiment(:default) { } assert_instance_of Vanity::Experiment::AbTest, experiment(:default) end it "requires at least two alternatives per experiment" do assert_raises RuntimeError do experiment :none, type: :ab_test do + alternatives [] end end assert_raises RuntimeError do @@ -30,29 +27,12 @@ class AbTestTest < MiniTest::Spec end end - it "complains if alternatives changed for stored definition" do - experiment :default do - true_false - end - - new_playground - assert_raises RuntimeError do - experiment :default do - alternatives "foo", "bar" - end - end - - new_playground - experiment :default do - true_false - end - end - it "returns the same alternative consistently from choice" do experiment :foobar do alternatives "foo", "bar" end - assert_match /foo|bar/, value = experiment(:foobar).choice(Vanity.identity) + assert value = experiment(:foobar).choice(Vanity.identity) + assert_match /foo|bar/, value 1000.times do assert_equal value, experiment(:foobar).choice(Vanity.identity) end @@ -104,8 +84,4 @@ class AbTestTest < MiniTest::Spec totals = experiment(:foobar).measure assert_equal ids.size, totals.inject(0) { |a,(k,v)| a + v[:conversions] } end - - after do - nuke_playground - end end test/ab_test_test.rb @@ -49,34 +49,27 @@ class PlaygroundTest < MiniTest::Spec it "returns experiment defined in file" do playground = class << Vanity.playground ; self ; end playground.send :define_method, :require do |file| - Vanity.playground.define "Green Button" do - true_false - end + Vanity.playground.define("Green Button") { } end assert_equal "green_button", experiment("Green button").name end it "can define and access experiment using symbol" do - assert green = experiment("Green Button") { true_false } + assert green = experiment("Green Button") { } assert_equal green, experiment(:green_button) - assert red = experiment(:red_button) { true_false } + assert red = experiment(:red_button) { } assert_equal red, experiment("Red Button") end it "detect and fail when defining the same experiment twice" do - experiment "Green Button" do - true_false - end + experiment("Green Button") { } assert_raises RuntimeError do - experiment :green_button do - true_false - end + experiment(:green_button) { } end end it "evalutes definition block when creating experiment" do experiment :green_button do - true_false expects(:xmts).returns("x") end assert_equal "x", experiment(:green_button).xmts @@ -84,32 +77,13 @@ class PlaygroundTest < MiniTest::Spec it "saves experiments after defining it" do experiment :green_button do - true_false expects(:save) end end - it "loads experiment if one already exists" do - experiment :green_button do - true_false - @xmts = "x" - end - Vanity.instance_variable_set :@playground, Vanity::Playground.new - experiment :green_button do - expects(:xmts).returns(@xmts) - end - assert_equal "x", experiment(:green_button).xmts - end - it "uses playground namespace in experiment" do - experiment :green_button do - true_false - end + experiment(:green_button) { } assert_equal "#{@namespace}:experiments:green_button", experiment(:green_button).send(:key) assert_equal "#{@namespace}:experiments:green_button:participants", experiment(:green_button).send(:key, "participants") end - - after do - nuke_playground - end end test/playground_test.rb @@ -108,9 +108,7 @@ end class AbTestHelpersTest < ActionController::TestCase tests AbTestHelpersController def setup - experiment :simple_ab do - true_false - end + experiment(:simple_ab) { } end def test_fail_if_no_experiment test/rails_test.rb @@ -36,6 +36,10 @@ class MiniTest::Unit::TestCase Vanity.playground.experiment(name) end end + + def teardown + nuke_playground + end end ActionController::Routing::Routes.draw do |map| test/test_helper.rb @@ -12,9 +12,9 @@ Gem::Specification.new do |spec| spec.executable = "vanity" spec.has_rdoc = true - spec.extra_rdoc_files = 'README.rdoc', 'CHANGELOG' - spec.rdoc_options = '--title', 'Vanity', '--main', 'README.rdoc', - '--webcvs', 'http://github.com/assaf/vanity' + spec.extra_rdoc_files = "README.rdoc", "CHANGELOG" + spec.rdoc_options = "--title", "Vanity #{spec.version}", "--main", "README.rdoc", + "--webcvs", "http://github.com/assaf/#{spec.name}" spec.add_dependency "redis", "0.1" end vanity.gemspec db38a01e7aa8a5642b0f746e6a8b9d05b300e02f Assaf Arkin assaf@labnotes.org http://github.com/assaf/vanity/commit/db700f0bd2d80cb7b71ff6ced983cf079f4c6fea db700f0bd2d80cb7b71ff6ced983cf079f4c6fea 2009-11-06T15:29:32-08:00 2009-11-06T15:29:32-08:00 A/B tests: alternatives no longer stored in Redis. A/B tests: default test is true/false. Experiment only stores created_at timestamps. Added Rake test task. 0e7d5943d104e7f1a69929c6fbf98b0122b645d8 Assaf Arkin assaf@labnotes.org