Permalink
Browse files

Add support for an exponentially decaying sample for histogram

  • Loading branch information...
1 parent 6e8cd61 commit c96598ffa38223fe73fa0dc619a9495a0e78a427 @eric committed Feb 28, 2012
View
@@ -2,6 +2,10 @@ source :rubygems
gemspec
+group :development do
+ gem 'perftools.rb'
+end
+
group :test do
gem 'turn'
end
View
@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'benchmark'
+require 'metriks'
+
+
+def fib(n)
+ n < 2 ? n : fib(n-1) + fib(n-2)
+end
+
+uniform_timer = Metriks::Timer.new(Metriks::Histogram.new_uniform)
+exponential_timer = Metriks::Timer.new(Metriks::Histogram.new_exponentially_decaying)
+
+fib_times = ARGV[0] ? ARGV[0].to_i : 10
+iter = ARGV[1] ? ARGV[1].to_i : 100000
+
+puts "fib(#{fib_times}): #{iter} iterations"
+puts "-" * 50
+
+plain = Benchmark.realtime do
+ for i in 1..iter
+ fib(fib_times)
+ end
+end
+
+puts "%15s: %f secs %f secs/call" % [ 'plain', plain, plain / iter ]
+
+uniform = Benchmark.realtime do
+ for i in 1..iter
+ uniform_timer.time do
+ fib(fib_times)
+ end
+ end
+end
+
+puts "%15s: %f secs %f secs/call -- %.1f%% slower than plain (%f secs/call)" % [
+ 'uniform', uniform, uniform / iter,
+ (uniform - plain) / plain * 100 ,
+ (uniform - plain) / iter,
+]
+
+exponential = Benchmark.realtime do
+ for i in 1..iter
+ exponential_timer.time do
+ fib(fib_times)
+ end
+ end
+end
+
+puts "%15s: %f secs %f secs/call -- %.1f%% slower than plain (%f secs/call) -- %.1f%% slower than uniform (%f secs/call)" % [
+ 'exponential', exponential, exponential / iter,
+ (exponential - plain) / plain * 100 ,
+ (exponential - plain) / iter,
+ (exponential - uniform) / uniform * 100 ,
+ (exponential - uniform) / iter
+]
+
+
@@ -0,0 +1,80 @@
+require 'atomic'
+require 'rbtree'
+require 'metriks/snapshot'
+
+module Metriks
+ class ExponentiallyDecayingSample
+ RESCALE_THRESHOLD = 60 * 60 # 1 hour
+
+ def initialize(reservoir_size, alpha)
+ @values = RBTree.new
+ @count = Atomic.new(0)
+ @next_scale_time = Atomic.new(0)
+ @alpha = alpha
+ @reservoir_size = reservoir_size
+ @mutex = Mutex.new
+ clear
+ end
+
+ def clear
+ @mutex.synchronize do
+ @values.clear
+ @count.value = 0
+ @next_scale_time.value = Time.now + RESCALE_THRESHOLD
+ @start_time = Time.now
+ end
+ end
+
+ def size
+ count = @count.value
+ count < @reservoir_size ? count : @reservoir_size
+ end
+
+ def snapshot
+ @mutex.synchronize do
+ Snapshot.new(@values.values)
+ end
+ end
+
+ def update(value, timestamp = Time.now)
+ @mutex.synchronize do
+ priority = Math.exp(timestamp - @start_time) / rand
+ new_count = @count.update { |v| v + 1 }
+ if new_count <= @reservoir_size
+ @values[priority] = value
+ else
+ first_priority = @values.first[0]
+ if first_priority < priority
+ unless @values[priority]
+ @values[priority] = value
+
+ until @values.delete(first_priority)
+ first_priority = @values.first[0]
+ end
+ end
+ end
+ end
+ end
+
+ now = Time.new
+ next_time = @next_scale_time.value
+ if now >= next_time
+ rescale(now, next_time)
+ end
+ end
+
+ def rescale(now, next_time)
+ if @next_scale_time.compare_and_swap(next_time, now + RESCALE_THRESHOLD)
+ @mutex.synchronize do
+ old_start_time = @start_time
+ @start_time = Time.now
+ keys = @values.keys
+ keys.each do |key|
+ value = @values.delete(key)
+ @values[key* Math.exp(-@alpha * (@start_time - old_start_time))] = value
+ end
+ end
+ end
+ end
+ end
+end
@@ -1,5 +1,6 @@
require 'atomic'
require 'metriks/uniform_sample'
+require 'metriks/exponentially_decaying_sample'
module Metriks
class Histogram
@@ -10,6 +11,10 @@ def self.new_uniform
new(Metriks::UniformSample.new(DEFAULT_SAMPLE_SIZE))
end
+ def self.new_exponentially_decaying
+ new(Metriks::ExponentiallyDecayingSample.new(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA))
+ end
+
def initialize(sample)
@sample = sample
@count = Atomic.new(0)
@@ -37,6 +42,10 @@ def update(value)
update_variance(value)
end
+ def snapshot
+ @sample.snapshot
+ end
+
def count
@count.value
end
@@ -51,24 +51,29 @@ def write
:min, :max, :mean, :stddev,
:one_minute_utilization, :five_minute_utilization,
:fifteen_minute_utilization, :mean_utilization,
+ ], [
+ :median, :get_95th_percentile
]
when Metriks::Timer
log_metric name, 'timer', metric, [
:count, :one_minute_rate, :five_minute_rate,
:fifteen_minute_rate, :mean_rate,
:min, :max, :mean, :stddev
+ ], [
+ :median, :get_95th_percentile
]
end
end
end
def extract_from_metric(metric, *keys)
keys.flatten.collect do |key|
- [ { key => metric.send(key) } ]
+ name = key.to_s.gsub(/^get_/, '')
+ [ { name => metric.send(key) } ]
end
end
- def log_metric(name, type, metric, *keys)
+ def log_metric(name, type, metric, keys, snapshot_keys = [])
message = []
message << @prefix if @prefix
@@ -78,6 +83,11 @@ def log_metric(name, type, metric, *keys)
message << { :type => type }
message += extract_from_metric(metric, keys)
+ unless snapshot_keys.empty?
+ snapshot = metric.snapshot
+ message += extract_from_metric(snapshot, snapshot_keys)
+ end
+
@logger.add(@log_level, format_message(message))
end
@@ -0,0 +1,56 @@
+
+class Snapshot
+ MEDIAN_Q = 0.5
+ P75_Q = 0.75
+ P95_Q = 0.95
+ P98_Q = 0.98
+ P99_Q = 0.99
+ P999_Q = 0.999
+
+ def initialize(values)
+ @values = values.sort
+ end
+
+ def value(quantile)
+ raise ArgumentError, "quantile must be between 0.0 and 1.0" if quantile < 0.0 || quantile > 1.0
+
+ return 0.0 if @values.empty?
+
+ pos = quantile * (@values.length + 1)
+
+ return @values.first if pos < 1
+ return @values.last if pos >= @values.length
+
+ lower = @values[pos.to_i - 1]
+ upper = @values[pos.to_i]
+ lower + (pos - pos.floor) + (upper - lower)
+ end
+
+ def size
+ @values.length
+ end
+
+ def median
+ value(MEDIAN_Q)
+ end
+
+ def get_75th_percentile
+ value(P75_Q)
+ end
+
+ def get_95th_percentile
+ value(P95_Q)
+ end
+
+ def get_98th_percentile
+ value(P98_Q)
+ end
+
+ def get_99th_percentile
+ value(P99_Q)
+ end
+
+ def get_999th_percentile
+ value(P999_Q)
+ end
+end
@@ -18,9 +18,9 @@ def stop
end
end
- def initialize
+ def initialize(histogram = Metriks::Histogram.new_uniform)
@meter = Metriks::Meter.new
- @histogram = Metriks::Histogram.new_uniform
+ @histogram = histogram
end
def clear
@@ -50,6 +50,10 @@ def time(callable = nil, &block)
end
end
+ def snapshot
+ @histogram.snapshot
+ end
+
def count
@histogram.count
end
@@ -1,4 +1,5 @@
require 'atomic'
+require 'metriks/snapshot'
module Metriks
class UniformSample
@@ -19,6 +20,10 @@ def size
count > @values.length ? @values.length : count
end
+ def snapshot
+ Snapshot.new(@values.slice(0, size))
+ end
+
def update(value)
new_count = @count.update { |v| v + 1 }
View
@@ -44,6 +44,7 @@ Gem::Specification.new do |s|
## that are needed for an end user to actually USE your code.
s.add_dependency('atomic', ["~> 1.0"])
s.add_dependency('hitimes', [ "~> 1.1"])
+ s.add_dependency('rbtree', [ "~> 0.3" ])
## List your development dependencies here. Development dependencies are
## those that are only needed during development
Oops, something went wrong.

0 comments on commit c96598f

Please sign in to comment.