From 432f95eab4c647ad6c33c34d1520706aad1d3735 Mon Sep 17 00:00:00 2001 From: Zach Millman Date: Sun, 20 Jan 2013 14:17:24 -0800 Subject: [PATCH 1/6] Allow the rubber:start task to start multiple instances --- lib/rubber/recipes/rubber/instances.rb | 57 +++++++++++++++++++------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/lib/rubber/recipes/rubber/instances.rb b/lib/rubber/recipes/rubber/instances.rb index 97aac3101..23210cda8 100644 --- a/lib/rubber/recipes/rubber/instances.rb +++ b/lib/rubber/recipes/rubber/instances.rb @@ -88,9 +88,11 @@ Start the EC2 instance for the give ALIAS DESC required_task :start do - instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true) + instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true) + + aliases = Rubber::Util::parse_aliases(instance_aliases) ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI - start_instance(instance_alias) + start_instances(aliases) end desc <<-DESC @@ -484,6 +486,45 @@ def stop_instance(instance_alias) cloud.stop_instance(instance_item.instance_id) end + # Starts the given ec2 instances. Note that this operation only works for instances that use an EBS volume for the root + # device, that are not spot instances, and that are already stopped. + def start_instances(aliases) + start_threads = [] + + instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]} + instance_items = filter_for_startable_stoppable_instances(instance_items) + + if instance_items.size == 0 + fatal "No instances to start!" + else + human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (instance_item.instance_id)"}.join(', ') + value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") + fatal("Exiting", 0) if value != "yes" + + instance_items.each |instance_item| + start_instance(instance_item.name) + + # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values. + start_threads << Thread.new do + while ! refresh_instance(instance_item.name) + sleep 1 + end + end + end + + start_threads.each {|t| t.join } + + print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to start" + while true do + print "." + sleep 2 + break unless start_threads.any(&:alive?) + end + + post_refresh + end + end + # Starts the given ec2 instance. Note that this operation only works for instances that use an EBS volume for the root # device, that are not spot instances, and that are already stopped. def start_instance(instance_alias) @@ -494,21 +535,9 @@ def start_instance(instance_alias) env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name) - value = Capistrano::CLI.ui.ask("About to START #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") - fatal("Exiting", 0) if value != "yes" - logger.info "Starting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}" cloud.start_instance(instance_item.instance_id) - - # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values. - print "Waiting for instance to start" - while true do - print "." - sleep 2 - - break if refresh_instance(instance_alias) - end end # delete from ~/.ssh/known_hosts all lines that begin with ec2- or instance_alias From d23ad3c7adb54ee033384a92d5b5f69a499a57ee Mon Sep 17 00:00:00 2001 From: Zach Millman Date: Sun, 20 Jan 2013 14:24:38 -0800 Subject: [PATCH 2/6] Fix syntax error in start_instances() --- lib/rubber/recipes/rubber/instances.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rubber/recipes/rubber/instances.rb b/lib/rubber/recipes/rubber/instances.rb index 23210cda8..b1e306ffe 100644 --- a/lib/rubber/recipes/rubber/instances.rb +++ b/lib/rubber/recipes/rubber/instances.rb @@ -501,7 +501,7 @@ def start_instances(aliases) value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") fatal("Exiting", 0) if value != "yes" - instance_items.each |instance_item| + instance_items.each do |instance_item| start_instance(instance_item.name) # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values. From dd90c5f7417aa4ca2c72a7bac09e64bde1afc989 Mon Sep 17 00:00:00 2001 From: Zach Millman Date: Sun, 20 Jan 2013 14:29:05 -0800 Subject: [PATCH 3/6] Clean up broken method call --- lib/rubber/recipes/rubber/instances.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/rubber/recipes/rubber/instances.rb b/lib/rubber/recipes/rubber/instances.rb index b1e306ffe..75b1745dc 100644 --- a/lib/rubber/recipes/rubber/instances.rb +++ b/lib/rubber/recipes/rubber/instances.rb @@ -492,7 +492,6 @@ def start_instances(aliases) start_threads = [] instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]} - instance_items = filter_for_startable_stoppable_instances(instance_items) if instance_items.size == 0 fatal "No instances to start!" From a0c393002713ae804710811ed63cde3a162ea27d Mon Sep 17 00:00:00 2001 From: Zach Millman Date: Sun, 20 Jan 2013 14:35:27 -0800 Subject: [PATCH 4/6] Fix threading for start_instances() --- lib/rubber/recipes/rubber/instances.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/rubber/recipes/rubber/instances.rb b/lib/rubber/recipes/rubber/instances.rb index 75b1745dc..5bf783612 100644 --- a/lib/rubber/recipes/rubber/instances.rb +++ b/lib/rubber/recipes/rubber/instances.rb @@ -496,15 +496,15 @@ def start_instances(aliases) if instance_items.size == 0 fatal "No instances to start!" else - human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (instance_item.instance_id)"}.join(', ') + human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ') value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") fatal("Exiting", 0) if value != "yes" instance_items.each do |instance_item| - start_instance(instance_item.name) - - # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values. start_threads << Thread.new do + start_instance(instance_item.name) + + # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values. while ! refresh_instance(instance_item.name) sleep 1 end @@ -517,7 +517,7 @@ def start_instances(aliases) while true do print "." sleep 2 - break unless start_threads.any(&:alive?) + break unless start_threads.any?(&:alive?) end post_refresh From 7dd6ea221123216955b237f525d8178bf3037cd3 Mon Sep 17 00:00:00 2001 From: Zach Millman Date: Sun, 20 Jan 2013 15:01:55 -0800 Subject: [PATCH 5/6] Allow rubber:stop to manage multiple instances --- lib/rubber/recipes/rubber/instances.rb | 54 ++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/lib/rubber/recipes/rubber/instances.rb b/lib/rubber/recipes/rubber/instances.rb index 5bf783612..f87d05c16 100644 --- a/lib/rubber/recipes/rubber/instances.rb +++ b/lib/rubber/recipes/rubber/instances.rb @@ -79,9 +79,11 @@ Stop the EC2 instance for the give ALIAS DESC required_task :stop do - instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true) + instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true) + + aliases = Rubber::Util::parse_aliases(instance_aliases) ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI - stop_instance(instance_alias) + stop_instances(aliases) end desc <<-DESC @@ -467,22 +469,60 @@ def reboot_instance(instance_alias, force=false) cloud.reboot_instance(instance_item.instance_id) end + + # Stops the given ec2 instances. Note that this operation only works for instances that use an EBS volume for the root + # device and that are not spot instances. + def stop_instances(aliases) + stop_threads = [] + + instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]} + # TODO: validate instances here + + if instance_items.size == 0 + fatal "No instances to stop!" + else + human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ') + value = Capistrano::CLI.ui.ask("About to STOP #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") + fatal("Exiting", 0) if value != "yes" + + instance_items.each do |instance_item| + logger.info "Stopping instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}" + + stop_threads << Thread.new do + stop_instance(instance_item.name) + + stopped = false + while !stopped + sleep 1 + instance = cloud.describe_instances(instance_item.instance_id).first rescue {} + stopped = (instance[:state] == "stopped") + end + end + end + + print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to stop" + while true do + print "." + sleep 2 + break unless stop_threads.any?(&:alive?) + end + print "\n" + + stop_threads.each {|t| t.join } + end + end # Stops the given ec2 instance. Note that this operation only works for instances that use an EBS volume for the root # device and that are not spot instances. def stop_instance(instance_alias) instance_item = rubber_instances[instance_alias] + fatal "Instance does not exist: #{instance_alias}" if ! instance_item fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil? fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs') env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name) - value = Capistrano::CLI.ui.ask("About to STOP #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") - fatal("Exiting", 0) if value != "yes" - - logger.info "Stopping instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}" - cloud.stop_instance(instance_item.instance_id) end From c3e5e9bd706e6e0680461e00504667315a4682f2 Mon Sep 17 00:00:00 2001 From: Zach Millman Date: Sun, 20 Jan 2013 15:32:43 -0800 Subject: [PATCH 6/6] Improve logging and handling of instance validation --- lib/rubber/recipes/rubber/instances.rb | 148 ++++++++++++------------- 1 file changed, 70 insertions(+), 78 deletions(-) diff --git a/lib/rubber/recipes/rubber/instances.rb b/lib/rubber/recipes/rubber/instances.rb index f87d05c16..1db324cb4 100644 --- a/lib/rubber/recipes/rubber/instances.rb +++ b/lib/rubber/recipes/rubber/instances.rb @@ -476,107 +476,99 @@ def stop_instances(aliases) stop_threads = [] instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]} - # TODO: validate instances here + instance_items = aliases.collect do |instance_alias| + instance_item = rubber_instances[instance_alias] + + fatal "Instance does not exist: #{instance_alias}" if ! instance_item + fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil? + fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs') + + instance_item + end - if instance_items.size == 0 - fatal "No instances to stop!" - else - human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ') - value = Capistrano::CLI.ui.ask("About to STOP #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") - fatal("Exiting", 0) if value != "yes" + # Get user confirmation + human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ') + value = Capistrano::CLI.ui.ask("About to STOP #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") + fatal("Exiting", 0) if value != "yes" - instance_items.each do |instance_item| - logger.info "Stopping instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}" + instance_items.each do |instance_item| + logger.info "Stopping instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}" + + stop_threads << Thread.new do + env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name) + + cloud.stop_instance(instance_item.instance_id) - stop_threads << Thread.new do - stop_instance(instance_item.name) - - stopped = false - while !stopped - sleep 1 - instance = cloud.describe_instances(instance_item.instance_id).first rescue {} - stopped = (instance[:state] == "stopped") - end + stopped = false + while !stopped + sleep 1 + instance = cloud.describe_instances(instance_item.instance_id).first rescue {} + stopped = (instance[:state] == "stopped") end end + end - print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to stop" - while true do - print "." - sleep 2 - break unless stop_threads.any?(&:alive?) - end - print "\n" - - stop_threads.each {|t| t.join } + print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to stop" + while true do + print "." + sleep 2 + break unless stop_threads.any?(&:alive?) end - end - - # Stops the given ec2 instance. Note that this operation only works for instances that use an EBS volume for the root - # device and that are not spot instances. - def stop_instance(instance_alias) - instance_item = rubber_instances[instance_alias] - - fatal "Instance does not exist: #{instance_alias}" if ! instance_item - fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil? - fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs') - - env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name) - - cloud.stop_instance(instance_item.instance_id) + print "\n" + + stop_threads.each(&:join) end # Starts the given ec2 instances. Note that this operation only works for instances that use an EBS volume for the root # device, that are not spot instances, and that are already stopped. def start_instances(aliases) start_threads = [] + refresh_threads = [] - instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]} - - if instance_items.size == 0 - fatal "No instances to start!" - else - human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ') - value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") - fatal("Exiting", 0) if value != "yes" + instance_items = aliases.collect do |instance_alias| + instance_item = rubber_instances[instance_alias] + + fatal "Instance does not exist: #{instance_alias}" if ! instance_item + fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil? + fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs') + + instance_item + end - instance_items.each do |instance_item| - start_threads << Thread.new do - start_instance(instance_item.name) - - # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values. + # Get user confirmation + human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ') + value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ") + fatal("Exiting", 0) if value != "yes" + + instance_items.each do |instance_item| + logger.info "Starting instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}" + + start_threads << Thread.new do + env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name) + + cloud.start_instance(instance_item.instance_id) + + # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values. + refresh_threads << Thread.new do while ! refresh_instance(instance_item.name) sleep 1 end end end + end - start_threads.each {|t| t.join } - - print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to start" - while true do - print "." - sleep 2 - break unless start_threads.any?(&:alive?) - end - - post_refresh + print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to start" + while true do + print "." + sleep 2 + break unless start_threads.any?(&:alive?) end - end - - # Starts the given ec2 instance. Note that this operation only works for instances that use an EBS volume for the root - # device, that are not spot instances, and that are already stopped. - def start_instance(instance_alias) - instance_item = rubber_instances[instance_alias] - fatal "Instance does not exist: #{instance_alias}" if ! instance_item - fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil? - fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs') - - env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name) - logger.info "Starting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}" - - cloud.start_instance(instance_item.instance_id) + start_threads.each(&:join) + refresh_threads.each(&:join) + + # Static IPs, DNS, etc. need to be set up for the started instances + post_refresh end # delete from ~/.ssh/known_hosts all lines that begin with ec2- or instance_alias