diff --git a/lib/vanity.rb b/lib/vanity.rb index 3e097562..31cf62ed 100644 --- a/lib/vanity.rb +++ b/lib/vanity.rb @@ -1,6 +1,7 @@ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "../vendor/redis-0.1/lib") require "redis" require "openssl" +require "date" # All the cool stuff happens in other places: # - Vanity::Helpers @@ -18,6 +19,7 @@ module Version end require "vanity/playground" +require "vanity/metric" require "vanity/experiment/base" require "vanity/experiment/ab_test" require "vanity/rails" if defined?(Rails) diff --git a/lib/vanity/metric.rb b/lib/vanity/metric.rb new file mode 100644 index 00000000..1e9ada01 --- /dev/null +++ b/lib/vanity/metric.rb @@ -0,0 +1,113 @@ +module Vanity + + # A metric is an object with a method called values, which accepts two + # arguments, start data and end date, and returns an array of measurements. + # + # A metric can also respons to additional methods (track!, bounds, etc). + # This class implements a metric, use this as reference to the methods you + # can implement in your own metric. + # + # Or just use this metric implementation. It's fast and fully functional. + # + # Startup metrics for pirates: AARRR stands for Acquisition, Activation, + # Retention, Referral and Revenue. + # http://500hats.typepad.com/500blogs/2007/09/startup-metrics.html + class Metric + + class << self + + # Helper method to return title for a metric. + # + # A metric object may have a +title+ method that returns a short + # descriptive title. It may also have no title, or no +title+ method, in + # which case the metric identifier will do. + # + # Example: + # Vanity.playground.metrics.map { |id, metric| Vanity::Metric.title(id, metric) } + def title(id, metric) + metric.respond_to?(:title) && metric.title || id.to_s.capitalize.gsub(/_+/, " ") + end + + # Helper method to return description for a metric. + # + # A metric object may have a +description+ method that returns a detailed + # description. It may also have no description, or no +description+ + # method, in which case return +nil+. + # + # Example: + # puts Vanity::Metric.description(metric) + def description(metric) + metric.description if metric.respond_to?(:description) + end + + # Helper method to return bounds for a metric. + # + # A metric object may have a +bounds+ method that returns lower and upper + # bounds. It may also have no bounds, or no +bounds+ # method, in which + # case we return +[nil, nil]+. + # + # Example: + # upper = Vanity::Metric.bounds(metric).last + def bounds(metric) + metric.respond_to?(:bounds) && metric.bounds || [nil, nil] + end + + end + + def initialize(playground, id) + @playground = playground + @id = id + @hooks = [] + end + + # All metrics implement this value. Given two arguments, a start date and + # an end date, it returns an array of measurements. + def values(to, from) + @playground.redis.mget((to.to_date..from.to_date).map { |date| "metrics:#{id}:#{date}:count" }).map(&:to_i) + end + + # This method returns the acceptable bounds of a metric as an array with + # two values: low and high. Use nil for unbounded. + # + # Alerts are created when metric values exceed their bounds. For example, + # a metric of user registration can use historical data to calculate + # expected range of new registration for the next day. If actual metric + # falls below the expected range, it could indicate registration process is + # broken. Going above higher bound could trigger opening a Champagne + # bottle. + # + # The default implementation returns +nil+. + def bounds + end + + # Metric identifier. + attr_accessor :id + + # Metric title. + attr_accessor :title + + # Metric description. + attr_accessor :description + + # Called to track an action associated with this metric. + def track!(vanity_id) + timestamp = Time.now + @playground.redis.incr "metrics:#{id}:#{timestamp.to_date}:count" + @playground.logger.info "vanity tracked #{title || id}" + @hooks.each do |hook| + hook.call id, timestamp, vanity_id + end + end + + # Metric definitions use this to introduce tracking hook. The hook is + # called with three arguments: metric id, timestamp and vanity identity. + # + # For example: + # hook do |metric_id, timestamp, vanity_id| + # syslog.info metric_id + # end + def hook(&block) + @hooks << block + end + end +end diff --git a/lib/vanity/playground.rb b/lib/vanity/playground.rb index 5562b4b4..664d63b9 100644 --- a/lib/vanity/playground.rb +++ b/lib/vanity/playground.rb @@ -27,9 +27,12 @@ class Playground # Created new Playground. Unless you need to, use the global Vanity.playground. def initialize @experiments = {} + @metrics = {} @host, @port, @db = "127.0.0.1", 6379, 0 @namespace = "vanity:#{Vanity::Version::MAJOR}" @load_path = "experiments" + @logger = Logger.new(STDOUT) + @logger.level = Logger::ERROR end # Redis host name. Default is 127.0.0.1 @@ -117,6 +120,22 @@ class << self ; self ; end.send(:define_method, :redis) { redis } redis end + # Returns a metric (creating one if doesn't already exist). + def metric(id) + id = id.to_sym + @metrics[id] ||= Metric.new(self, id) + end + + # Returns hash of metrics (key is metric id). + def metrics + @metrics + end + + # Tracks an action associated with a metric. For example: + # Vanity.playground.track! :uploaded_video + def track!(id) + metric(id).track! Vanity.context.vanity_identity + end end @playground = Playground.new diff --git a/lib/vanity/rails/helpers.rb b/lib/vanity/rails/helpers.rb index 84bd573e..2cc04310 100644 --- a/lib/vanity/rails/helpers.rb +++ b/lib/vanity/rails/helpers.rb @@ -103,8 +103,8 @@ def ab_test(name, &block) # track! :call_to_action # Acccount.create! params[:account] # end - def track!(name, *args) - Vanity.playground.experiment(name).track! *args + def track!(name) + Vanity.playground.track! name end end end diff --git a/test/metric_test.rb b/test/metric_test.rb new file mode 100644 index 00000000..6ef704b9 --- /dev/null +++ b/test/metric_test.rb @@ -0,0 +1,141 @@ +require "test/test_helper" + +class MetricTest < MiniTest::Unit::TestCase + def setup + super + Vanity.context = mock("Context") + Vanity.context.stubs(:vanity_identity).returns(rand) + end + + # -- Via the playground -- + + def test_playground_creates_metric_on_demand + assert metric = Vanity.playground.metric(:on_demand) + assert_equal :on_demand, metric.id + end + + def test_playground_tracks_all_loaded_metrics + Vanity.playground.metric(:work) + Vanity.playground.metric(:play) + assert_includes Vanity.playground.metrics.keys, :play + assert_includes Vanity.playground.metrics.keys, :work + end + + def test_playground_tracking_creates_metric_on_demand + Vanity.playground.track! :on_demand + assert_includes Vanity.playground.metrics.keys, :on_demand + assert_respond_to Vanity.playground.metrics[:on_demand], :values + end + + + # -- Tracking -- + + def test_tracking_can_count + 4.times { Vanity.playground.track! :play } + 2.times { Vanity.playground.track! :work } + play = Vanity.playground.metric(:play).values(Date.today, Date.today).first + work = Vanity.playground.metric(:work).values(Date.today, Date.today).first + assert play = 2 * work + end + + def test_tracking_can_tell_the_time + Time.is (Date.today - 4).to_time do + 4.times { Vanity.playground.track! :play } + end + Time.is (Date.today - 2).to_time do + 2.times { Vanity.playground.track! :play } + end + 1.times { Vanity.playground.track! :play } + values = Vanity.playground.metric(:play).values(Date.today - 5, Date.today) + assert_equal [0,4,0,2,0,1], values + end + + + # -- Tracking and hooks -- + + def test_tracking_runs_hook + returns = 0 + Vanity.playground.metric(:many_happy_returns).hook do |metric_id, timestamp, vanity_id| + assert_equal :many_happy_returns, metric_id + assert_in_delta Time.now.to_i, timestamp.to_i, 1 + assert_equal Vanity.context.vanity_identity, vanity_id + returns += 1 + end + Vanity.playground.track! :many_happy_returns + assert_equal 1, returns + end + + def test_tracking_runs_multiple_hooks + returns = 0 + Vanity.playground.metric(:many_happy_returns).hook { returns += 1 } + Vanity.playground.metric(:many_happy_returns).hook { returns += 1 } + Vanity.playground.metric(:many_happy_returns).hook { returns += 1 } + Vanity.playground.track! :many_happy_returns + assert_equal 3, returns + end + + + # -- Title helper -- + + def test_title_for_metric_with_title + metric = Vanity.playground.metric(:bst) + metric.title = "Blood, sweat, tears" + assert_equal "Blood, sweat, tears", Vanity::Metric.title(:bst, metric) + end + + def test_title_for_metric_with_no_title + metric = Vanity.playground.metric(:bst) + assert_equal "Bst", Vanity::Metric.title(:bst, metric) + end + + def test_title_for_metric_with_no_title_attributes + metric = Object.new + assert_equal "Bst", Vanity::Metric.title(:bst, metric) + end + + def test_title_for_metric_with_compound_id + metric = Vanity.playground.metric(:blood_sweat_tears) + assert_equal "Blood sweat tears", Vanity::Metric.title(:blood_sweat_tears, metric) + end + + + # -- Description helper -- + + def test_description_for_metric_with_description + metric = Vanity.playground.metric(:bst) + metric.description = "I didn't say it will be easy" + assert_equal "I didn't say it will be easy", Vanity::Metric.description(metric) + end + + def test_description_for_metric_with_no_description + metric = Vanity.playground.metric(:bst) + assert_nil Vanity::Metric.description(metric) + end + + def test_description_for_metric_with_no_description_method + metric = Object.new + assert_nil Vanity::Metric.description(metric) + end + + + # -- Metric bounds -- + + def test_bounds_helper_for_metric_with_bounds + metric = Vanity.playground.metric(:eggs) + metric.instance_eval do + def bounds ; [6,12] ; end + end + assert_equal [6,12], Vanity::Metric.bounds(metric) + end + + def test_bounds_helper_for_metric_with_no_bounds + metric = Vanity.playground.metric(:sky_is_limit) + assert_equal [nil, nil], Vanity::Metric.bounds(metric) + end + + def test_bounds_helper_for_metric_with_no_bounds_method + metric = Object.new + assert_equal [nil, nil], Vanity::Metric.bounds(metric) + end + +end diff --git a/test/mock_redis.rb b/test/mock_redis.rb new file mode 100644 index 00000000..4f77b17c --- /dev/null +++ b/test/mock_redis.rb @@ -0,0 +1,62 @@ +# The Redis you should never use in production. +class MockRedis + @@hash = {} + + def initialize(options) + end + + def [](key) + @@hash[key] + end + + def []=(key, value) + @@hash[key] = value.to_s + end + + def del(key) + @@hash.delete key + end + + def setnx(key, value) + @@hash[key] = value.to_s unless @@hash.has_key?(key) + end + + def incr(key) + @@hash[key] = (@@hash[key].to_i + 1).to_s + end + + def mget(keys) + @@hash.values_at(*keys) + end + + def flushdb + @@hash.clear + end + + def sismember(key, value) + case set = @@hash[key] + when nil ; false + when Set ; set.member?(value) + else fail "Not a set" + end + end + + def sadd(key, value) + case set = @@hash[key] + when nil ; @@hash[key] = Set.new([value]) + when Set ; set.add value + else fail "Not a set" + end + end + + def scard(key) + case set = @@hash[key] + when nil ; 0 + when Set ; set.size + else fail "Not a set" + end + end +end + +# Use mock redis. +Object.const_set :Redis, MockRedis diff --git a/test/test_helper.rb b/test/test_helper.rb index 3e86043c..1f5de095 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -7,9 +7,11 @@ require "action_controller/test_case" require "initializer" require "lib/vanity/rails" +require "test/mock_redis" # <-- load this when you don't want to use Redis MiniTest::Unit.autorun class MiniTest::Unit::TestCase + # Call this on teardown. It wipes put the playground and any state held in it # (mostly experiments), resets vanity ID, and clears Redis of all experiments. def nuke_playground @@ -33,3 +35,24 @@ def teardown map.connect ':controller/:action/:id' end Rails.configuration = Rails::Configuration.new + + +# Time.now adapted from Jason Earl: +# http://jasonearl.com/blog/testing_time_dependent_code/index.html +def Time.now + @active || new +end + +# Set the time to be fake for a given block of code +def Time.is(new_time, &block) + if block_given? + begin + old_time, @active = @active, new_time + yield + ensure + @active = old_time + end + else + @active = new_time || Time.new + end +end