Skip to content
This repository
Browse code

Make confirm! and on_all_servers pruning more testable, and test them.

  • Loading branch information...
commit f6b206eb0261c220e6c7b98a95e0cf438fcfed73 1 parent 2195d7a
Stu Hood stuhood authored
2  Rakefile
@@ -17,7 +17,7 @@ Jeweler::Tasks.new do |gem|
17 17 gem.name = "gizzmo"
18 18 gem.summary = %Q{Gizzmo is a command-line client for managing gizzard clusters.}
19 19 gem.description = %Q{Gizzmo is a command-line client for managing gizzard clusters.}
20   - gem.email = "kmaxwell@twitter.com"
  20 + gem.email = "stuhood@twitter.com"
21 21 gem.homepage = "http://github.com/twitter/gizzmo"
22 22 gem.authors = ["Kyle Maxwell"]
23 23 end
24 lib/gizzard/commands.rb
@@ -3,17 +3,23 @@
3 3 require "digest/md5"
4 4
5 5 module Gizzard
6   - def Gizzard::confirm!(force=false, message="Continue?")
  6 + CONFIRM_OPTIONS = Hash[
  7 + 'y' => lambda { true },
  8 + 'n' => lambda { puts "Exiting"; exit 1 }
  9 + ].freeze
  10 +
  11 + # takes a map of 'character => lambda': if a character is defined, the result of
  12 + # executing the (no-arg) lambda is returned
  13 + def Gizzard::confirm!(force=false, message="Continue?", options=CONFIRM_OPTIONS, input=$stdin, output=$stdout)
7 14 return if force
  15 + opt_char_string = options.keys.join('/')
8 16 begin
9   - print "#{message} (y/n) "; $stdout.flush
10   - resp = $stdin.gets.chomp.downcase
11   - puts ""
12   - end while resp != 'y' && resp != 'n'
13   - if resp == 'n'
14   - puts "Exiting."
15   - exit
16   - end
  17 + output.print "#{message} (#{opt_char_string}) "
  18 + output.flush
  19 + resp = input.gets.chomp.downcase
  20 + output.puts ""
  21 + end while !options.has_key?(resp)
  22 + options[resp].call
17 23 end
18 24
19 25 class Command
108 lib/gizzard/nameserver.rb
@@ -89,6 +89,45 @@ class Nameserver
89 89 MAX_ATTEMPT_SECS = 30
90 90 PARALLELISM = 10
91 91
  92 + PRUNE_HOST_MSG =
  93 + "(r)etry on these hosts, (i)gnore these hosts for the remainder of the transform, (k)ill the process?"
  94 + PRUNE_HOST_OPTS = Hash[
  95 + 'r' => lambda { true },
  96 + 'i' => lambda { false },
  97 + 'k' => lambda { raise Exception.new("Killing transform.") }
  98 + ].freeze
  99 +
  100 + # given a list of all_clients, and a list of triples of (client, result, exception),
  101 + # ask the user how to handle the failed clients, and return a tuple of
  102 + # (all_clients, failed_clients_to_consider_successful). If the user does not want
  103 + # to proceed or there are no hosts to continue with, raises an exception.
  104 + def Nameserver.prune_hosts(force, operation_name, all_clients, failed_client_triples, input=$stdin, output=$stdout)
  105 + output.puts "#{failed_client_triples.size} of #{all_clients.size} clients " +
  106 + "failed to execute '#{operation_name}':"
  107 + failed_client_triples.each do |client, _, exception|
  108 + output.puts "\t#{client.get_host} failed with: #{exception}"
  109 + end
  110 + if force
  111 + raise Exception.new("Cannot proceed past exceptions while force=true: exiting.")
  112 + end
  113 + res = Gizzard::confirm!(false, PRUNE_HOST_MSG, PRUNE_HOST_OPTS, input, output)
  114 +
  115 + # we're still alive: user wanted to proceed, either by retrying failed hosts,
  116 + # or by pruning them
  117 + if res
  118 + # continue with full host list
  119 + [all_clients,[]]
  120 + else
  121 + # return an updated list
  122 + without_clients = failed_client_triples.map{|client, _, _| client }
  123 + res_all_clients = all_clients - without_clients
  124 + if res_all_clients.empty?
  125 + raise Exception.new("No viable clients remain: exiting.")
  126 + end
  127 + [res_all_clients, without_clients]
  128 + end
  129 + end
  130 +
92 131 attr_reader :hosts, :logfile, :dryrun, :framed
93 132 alias dryrun? dryrun
94 133
@@ -192,43 +231,46 @@ def validate_clients_or_raise
192 231 # executes the given block in parallel with a client for each server: in the face of failure,
193 232 # may return less results than there are clients
194 233 def on_all_servers(operation_name, &block)
195   - # fork into many threads, and then join with exception handling
196   - clients_and_threads = all_clients.map do |client|
197   - [client, Thread.new { Thread.current[:result] = block.call(client) }]
198   - end
199   - clients_and_results_or_exceptions = clients_and_threads.map do |client, thread|
200   - begin
201   - thread.join
202   - [client, thread[:result], nil]
203   - rescue Exception => e
204   - [client, nil, e]
  234 + remaining_clients = all_clients
  235 + successful_results = []
  236 + while true do
  237 + # fork into many threads, and then join with exception handling
  238 + clients_and_threads = remaining_clients.map do |client|
  239 + [client, Thread.new { Thread.current[:result] = block.call(client) }]
205 240 end
206   - end
207   -
208   - successful_clients, failed_clients =
209   - clients_and_results_or_exceptions.partition{|_, _, exception| exception.nil? }
210   - if failed_clients.size > 0
211   - # if there were failed clients, but the user would like to proceed anyway,
212   - # mutate @all_clients to remove the failed clients
213   - puts "#{failed_clients.size} of #{all_clients.size} clients failed to execute '#{operation_name}':"
214   - failed_clients.each do |client, _, exception|
215   - puts "\t#{client.get_host} failed with: #{exception}"
  241 + clients_and_results_or_exceptions = clients_and_threads.map do |client, thread|
  242 + begin
  243 + thread.join
  244 + [client, thread[:result], nil]
  245 + rescue Exception => e
  246 + [client, nil, e]
  247 + end
216 248 end
217   - if @force
218   - puts "Cannot proceed past exceptions while force=true: exiting."
219   - exit 1
  249 +
  250 + successful_clients, failed_clients =
  251 + clients_and_results_or_exceptions.partition{|_, _, exception| exception.nil? }
  252 + # collect successful results and remove successful clients
  253 + remaining_clients =
  254 + remaining_clients - successful_clients.map{|c, _, _| c }
  255 + successful_results =
  256 + successful_results + successful_clients.map{|_, r, _| r }
  257 +
  258 + if failed_clients.size > 0
  259 + begin
  260 + # if there were failed clients, but the user would like to proceed anyway,
  261 + # mutate all_clients and remaining_clients
  262 + all_clients, considered_successful_clients =
  263 + Nameserver.prune_hosts(@force, operation_name, all_clients, failed_clients)
  264 + remaining_clients =
  265 + remaining_clients - considered_successful_clients.map{|c, _, _| c }
  266 + rescue Exception => e
  267 + puts "Did not complete '#{operation_name}': " + e
  268 + exit 1
  269 + end
220 270 end
221   - Gizzard::confirm!(false, "Proceed without these hosts?")
222   - # we're still alive: user wanted to proceed
223   - @all_clients.reject!(failed_clients.map{|client, _, _| client })
224   - end
225   - if @all_clients.size < 1
226   - puts "No viable clients remain: exiting."
227   - exit 1
228   - end
229 271
230   - # return only successful results
231   - successful_clients.map{|_, result, _| result }
  272 + return successful_results if remaining_clients.empty?
  273 + end
232 274 end
233 275
234 276 def client
13 test/gizzmo_spec.rb
@@ -16,6 +16,19 @@ def fuzzily(str)
16 16 @nameserver_db = nil
17 17 end
18 18
  19 + describe "confirm!" do
  20 + it "differentiates between inputs" do
  21 + opt = Hash['y' => lambda {|| 'y' }]
  22 + output = sio()
  23 + Gizzard::confirm!(false, "Blah?", opt, sio("n\ny\n"), output).should == 'y'
  24 + output.string.should == "Blah? (y) \nBlah? (y) \n"
  25 + end
  26 +
  27 + def sio(string="")
  28 + StringIO.new(string)
  29 + end
  30 + end
  31 +
19 32 describe "basic manipulation commands" do
20 33 describe "create" do
21 34 it "creates a single shard" do
39 test/nameserver_spec.rb
@@ -101,6 +101,45 @@ def parse_should(hostname, table_prefix)
101 101 it "works..."
102 102 end
103 103
  104 + describe "prune_hosts" do
  105 + class Client
  106 + def initialize(host)
  107 + @host = host
  108 + end
  109 +
  110 + def get_host
  111 + @host
  112 + end
  113 + end
  114 +
  115 + one = Client.new(1)
  116 + two = Client.new(2)
  117 + all = [one, two]
  118 + failed = [[one, "Result", "Exception"]]
  119 +
  120 + it "ignores a failed client" do
  121 + prune("i\n", all, failed).should == [[two], [one]]
  122 + end
  123 +
  124 + it "retries a failed client" do
  125 + prune("r\n", all, failed).should == [all, []]
  126 + end
  127 +
  128 + it "fails when no more hosts" do
  129 + expect { prune("k\n", all, failed) }.should raise_error
  130 + end
  131 +
  132 + it "fails when requested" do
  133 + expect { prune("k\n", all, failed) }.should raise_error
  134 + end
  135 +
  136 + def prune(instr, all, failed)
  137 + input = StringIO.new(instr)
  138 + output = StringIO.new("")
  139 + Gizzard::Nameserver.prune_hosts(false, "prune", all, failed, input, output)
  140 + end
  141 + end
  142 +
104 143 describe "reload_config" do
105 144 it "reloads config on every app server" do
106 145 mock(@client).reload_config

0 comments on commit f6b206e

Please sign in to comment.
Something went wrong with that request. Please try again.