Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Manage multiple instances with rubber:stop and rubber:start #255

Merged
merged 6 commits into from Jan 26, 2013
Merged

Manage multiple instances with rubber:stop and rubber:start #255

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
432f95e
Allow the rubber:start task to start multiple instances
zmillman Jan 20, 2013
d23ad3c
Fix syntax error in start_instances()
zmillman Jan 20, 2013
dd90c5f
Clean up broken method call
zmillman Jan 20, 2013
a0c3930
Fix threading for start_instances()
zmillman Jan 20, 2013
7dd6ea2
Allow rubber:stop to manage multiple instances
zmillman Jan 20, 2013
c3e5e9b
Improve logging and handling of instance validation
zmillman Jan 20, 2013
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
134 changes: 97 additions & 37 deletions lib/rubber/recipes/rubber/instances.rb
View file
Open in desktop
Expand Up @@ -79,18 +79,22 @@
Stop the EC2 instance for the give ALIAS
DESC
required_task :stop do
instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true)

aliases = Rubber::Util::parse_aliases(instance_aliases)
ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
stop_instance(instance_alias)
stop_instances(aliases)
end

desc <<-DESC
Start the EC2 instance for the give ALIAS
DESC
required_task :start do
instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true)

aliases = Rubber::Util::parse_aliases(instance_aliases)
ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
start_instance(instance_alias)
start_instances(aliases)
end

desc <<-DESC
Expand Down Expand Up @@ -465,50 +469,106 @@ def reboot_instance(instance_alias, force=false)

cloud.reboot_instance(instance_item.instance_id)
end

# Stops the given ec2 instance. Note that this operation only works for instances that use an EBS volume for the root
# Stops the given ec2 instances. Note that this operation only works for instances that use an EBS volume for the root
# device and that are not spot instances.
def stop_instance(instance_alias)
instance_item = rubber_instances[instance_alias]
fatal "Instance does not exist: #{instance_alias}" if ! instance_item
fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil?
fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')

env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)

value = Capistrano::CLI.ui.ask("About to STOP #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}. Are you SURE [yes/NO]?: ")
def stop_instances(aliases)
stop_threads = []

instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]}
instance_items = aliases.collect do |instance_alias|
instance_item = rubber_instances[instance_alias]

fatal "Instance does not exist: #{instance_alias}" if ! instance_item
fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil?
fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')

instance_item
end

# Get user confirmation
human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ')
value = Capistrano::CLI.ui.ask("About to STOP #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ")
fatal("Exiting", 0) if value != "yes"

instance_items.each do |instance_item|
logger.info "Stopping instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}"

stop_threads << Thread.new do
env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)

logger.info "Stopping instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"

cloud.stop_instance(instance_item.instance_id)
cloud.stop_instance(instance_item.instance_id)

stopped = false
while !stopped
sleep 1
instance = cloud.describe_instances(instance_item.instance_id).first rescue {}
stopped = (instance[:state] == "stopped")
end
end
end

print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to stop"
while true do
print "."
sleep 2
break unless stop_threads.any?(&:alive?)
end
print "\n"

stop_threads.each(&:join)
end

# Starts the given ec2 instance. Note that this operation only works for instances that use an EBS volume for the root
# Starts the given ec2 instances. Note that this operation only works for instances that use an EBS volume for the root
# device, that are not spot instances, and that are already stopped.
def start_instance(instance_alias)
instance_item = rubber_instances[instance_alias]
fatal "Instance does not exist: #{instance_alias}" if ! instance_item
fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil?
fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')

env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)

value = Capistrano::CLI.ui.ask("About to START #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}. Are you SURE [yes/NO]?: ")
def start_instances(aliases)
start_threads = []
refresh_threads = []

instance_items = aliases.collect do |instance_alias|
instance_item = rubber_instances[instance_alias]

fatal "Instance does not exist: #{instance_alias}" if ! instance_item
fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil?
fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')

instance_item
end

# Get user confirmation
human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ')
value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}. Are you SURE [yes/NO]?: ")
fatal("Exiting", 0) if value != "yes"

logger.info "Starting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"

cloud.start_instance(instance_item.instance_id)

# Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values.
print "Waiting for instance to start"

instance_items.each do |instance_item|
logger.info "Starting instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}"

start_threads << Thread.new do
env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)

cloud.start_instance(instance_item.instance_id)

# Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values.
refresh_threads << Thread.new do
while ! refresh_instance(instance_item.name)
sleep 1
end
end
end
end

print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to start"
while true do
print "."
sleep 2

break if refresh_instance(instance_alias)
break unless start_threads.any?(&:alive?)
end

start_threads.each(&:join)
refresh_threads.each(&:join)

# Static IPs, DNS, etc. need to be set up for the started instances
post_refresh
end

# delete from ~/.ssh/known_hosts all lines that begin with ec2- or instance_alias
Expand Down