Permalink
Browse files

Add additional specs for load balancer; change backoff strategy to la…

…st failure time
  • Loading branch information...
1 parent dcffe44 commit 7a00863b042517599ab065957b7a7ba9c7c3d0af Will Fitzgerald committed Oct 30, 2012
Showing with 113 additions and 22 deletions.
  1. +17 −7 lib/wordnik/load_balancer.rb
  2. +96 −15 spec/load_balancer_spec.rb
@@ -19,8 +19,8 @@ class LoadBalancer
attr_accessor :current_host
def initialize(hosts)
- @all_hosts = hosts
- @hosts = @all_hosts
+ @all_hosts = hosts.clone
+ @hosts = @all_hosts.clone
@failed_hosts_table = {}
@current_host = nil
end
@@ -36,7 +36,7 @@ def inform_failure
#Wordnik.logger.debug "Informing failure about #{@current_host}. table: #{@failed_hosts_table.inspect}"
if @failed_hosts_table.include?(@current_host)
failures, failed_time = @failed_hosts_table[@current_host]
- @failed_hosts_table[@current_host] = [failures+1, failed_time]
+ @failed_hosts_table[@current_host] = [failures+1, Time.now.to_f]
else
@failed_hosts_table[@current_host] = [1, Time.now.to_f] # failure count, first failure time
end
@@ -57,15 +57,25 @@ def restore_failed_hosts_maybe
return if @failed_hosts_table.size == 0
@failed_hosts_table.each do |host, pair|
failures, failed_time = pair
- seconds_since_first_failure = (Time.now.to_f - failed_time)
- #Wordnik.logger.debug "Seconds since #{host}'s first failure: #{seconds_since_first_failure} compared to #{2**(failures-1)}"
+ n = Time.now.to_f
+ seconds_since_last_failure = (n - failed_time)
# exponential backoff, but try every hour...
- if (seconds_since_first_failure > [3600, 2**(failures-1)].min)
+ if (seconds_since_last_failure > [3600, 2**(failures-1)].min)
@hosts << host # give it a chance to succeed ...
- #Wordnik.logger.debug "Added #{host} to @hosts; now: #{@hosts}"
+ update_failed_time(host, n)
end
end
end
+
+ # mostly useful in mock testing...
+ def update_failed_time(host, time=Time.now)
+ if @failed_hosts_table.include? host
+ failures, _ = @failed_hosts_table[host]
+ @failed_hosts_table[host] = [failures, time.to_f]
+ else
+ @failed_hosts_table[host] = [1,time.to_f]
+ end
+ end
end
end
View
@@ -1,28 +1,109 @@
require 'spec_helper'
+require 'objspace' # ruby 1.9, I think
describe Wordnik::LoadBalancer do
- before(:each) do
- @hosts = ["alpha","beta","gamma","delta"].map{|x| "#{x}.wordnik.com"}
- end
+ before(:each) do
+ @hosts = ["alpha","beta","gamma","delta"].map{|x| "#{x}.wordnik.com"}
+ end
- describe "Load Balancer" do
+ describe "Load Balancer" do
it "allows creation with a list of hosts" do
- lb = Wordnik::LoadBalancer.new(@hosts)
- h = lb.host
- @hosts.should include(h)
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ h = lb.host
+ @hosts.should include(h)
end
it "returns different (random) hosts" do
- lb = Wordnik::LoadBalancer.new(@hosts)
- counts = Hash.new(0)
- 1.upto(1000) do
- h = lb.host
- counts[h] += 1
- end
- @hosts.each {|host| counts[host].should > 10}
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ counts = Hash.new(0)
+ 1.upto(1000) do
+ h = lb.host
+ counts[h] += 1
+ end
+ @hosts.each {|host| counts[host].should > 10}
+ end
+
+ it "should not leak memory" do
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ free = ObjectSpace.count_objects[:TOTAL]
+ 1000.times {h = lb.host}
+ free.should == ObjectSpace.count_objects[:TOTAL]
end
- it "should not leak memory"
+ describe "Success and failure modes" do
+ it "allows for a failure" do
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ h = lb.host
+ lb.inform_failure
+ lb.failed_hosts_table.size.should == 1
+ lb.hosts.should_not include(h)
+ lb.hosts.size.should == lb.all_hosts.size - 1
+ end
+
+ it "allows for two failures" do
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ h1 = lb.host
+ lb.inform_failure
+ h2 = lb.host
+ lb.inform_failure
+ lb.failed_hosts_table.size.should == 2
+ lb.hosts.should_not include(h1)
+ lb.hosts.should_not include(h2)
+ lb.hosts.size.should == lb.all_hosts.size - 2
+ end
+
+ it "should never leave hosts empty" do
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ @hosts.size.times{h = lb.host; lb.inform_failure}
+ lb.hosts.size.should == 1
+ end
+ it "allows for a subsequent success" do
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ h = lb.host
+ lb.inform_failure
+ lb.inform_success
+ lb.failed_hosts_table.size.should == 0
+ lb.hosts.should include(h)
+ lb.hosts.size.should == lb.all_hosts.size
+ end
+
+ it "does exponential back-off" do
+ lb = Wordnik::LoadBalancer.new(@hosts)
+ t = Time.now
+ h = lb.host
+ lb.inform_failure
+ lb.update_failed_time(h, t)
+ lb.update_failed_time(h, t-2) # pretend it happened two seconds ago
+ lb.restore_failed_hosts_maybe
+ lb.hosts.should include(h)
+ lb.hosts.size.should == lb.all_hosts.size
+ lb.inform_failure
+ lb.inform_failure
+ lb.update_failed_time(h, t-2) # pretend it happened two seconds ago
+ lb.restore_failed_hosts_maybe
+ lb.hosts.should_not include(h)
+ lb.hosts.size.should == lb.all_hosts.size - 1
+ lb.update_failed_time(h, t-4) # pretend it happened 4 seconds ago
+ lb.restore_failed_hosts_maybe
+ lb.hosts.should include(h)
+ lb.hosts.size.should == lb.all_hosts.size
+ # create a lot of failures
+ 1000.times {lb.inform_failure}
+ lb.update_failed_time(h, t-((59*60)+59)) # pretend it happened 59:59 minutes ago
+ lb.restore_failed_hosts_maybe
+ lb.hosts.should_not include(h)
+ lb.hosts.size.should == lb.all_hosts.size - 1
+ lb.update_failed_time(h, t-(60*60)) # pretend it happened 1 hour ago
+ lb.restore_failed_hosts_maybe
+ lb.hosts.should include(h)
+ lb.hosts.size.should == lb.all_hosts.size
+ lb.inform_success
+ lb.failed_hosts_table.size.should == 0
+ end
+ end
end
+
+
+
end

0 comments on commit 7a00863

Please sign in to comment.