Permalink
Browse files

add code to shut down slaves when instance exits

  • Loading branch information...
lstein committed Oct 31, 2012
1 parent e7d87f9 commit d2639dbb999667243995fb26453252bcd0e42747
Showing with 66 additions and 23 deletions.
  1. +27 −16 bin/gbrowse_add_slaves.pl
  2. +39 −7 bin/gbrowse_launch_aws_slaves.pl
View
@@ -29,6 +29,8 @@
This script does not actually provision new slaves. It is called after slaves
are provisioned (or deprovisioned) to update the configuration on the master and
restart the server.
+
+Use --set '' to disable the renderfarm completely.
USAGE
;
if (@ARGV && !@to_add && !@to_remove) {
@@ -39,33 +41,42 @@
my $render_conf = File::Spec->catfile($globals->config_base,'renderfarm.conf');
-e $render_conf or system 'touch',$render_conf;
my $conf = Bio::Graphics::FeatureFile->new(-file=>$render_conf) or die "Couldn't open $render_conf: $!";
-my $use_renderfarm = $conf->setting(general=>'renderfarm');
-my @remote_renderers = shellwords($conf->setting(general=>'remote renderer'));
-my %remote_renderers = map {$_=>1} @remote_renderers;
+my $use_renderfarm = $conf->setting(general=>'renderfarm');
+my @remote_renderers = shellwords($conf->setting(general=>'remote renderer'));
+my %remote_renderers = map {$_=>1} @remote_renderers;
+my %original_renderers = %remote_renderers;
for my $add (@to_add) {
+ next unless $add;
$remote_renderers{$add}++;
}
for my $sub (@to_remove) {
+ next unless $sub;
delete $remote_renderers{$sub};
}
if (@to_set) {
- %remote_renderers = map {$_=>1} @to_set;
+ %remote_renderers = map {$_=>1} grep {$_} @to_set;
}
-# write out
-$use_renderfarm = keys %remote_renderers > 0 ? 1 : 0;
+my $orig = join ' ',sort keys %original_renderers;
+my $new = join ' ',sort keys %remote_renderers;
+my $changed = $orig ne $new;
-open my $f,'>',"$render_conf.new" or die "Couldn't open $conf.new: $!";
-print $f "[GENERAl]\n";
-print $f "renderfarm = $use_renderfarm\n";
-print $f "remote renderer = \n";
-for my $s (keys %remote_renderers) {
- print $f "\t",$s,"\n" or die "Couldn't write: $!";
-}
-close $f or die "Couldn't write: $!";
-rename "$render_conf.new",$render_conf;
+if ($changed) {
+ # write out
+ $use_renderfarm = keys %remote_renderers > 0 ? 1 : 0;
-system "sudo /etc/init.d/apache2 graceful";
+ open my $f,'>',"$render_conf.new" or die "Couldn't open $conf.new: $!";
+ print $f "[GENERAl]\n";
+ print $f "renderfarm = $use_renderfarm\n";
+ print $f "remote renderer = \n";
+ for my $s (keys %remote_renderers) {
+ print $f "\t",$s,"\n" or die "Couldn't write: $!";
+ }
+ close $f or die "Couldn't write: $!";
+ rename "$render_conf.new",$render_conf;
+
+ system "sudo /etc/init.d/apache2 graceful";
+}
exit 0;
@@ -5,12 +5,25 @@
# to deal with it.
#
# All values are hard-coded as constants during this testing phase
+#
+# Need following security groups:
+# GBrowseMaster
+# allow inbound on 22 from all
+# allow inbound on 80 from all
+#
+# GBrowseSlave
+# allow inbound on 8101-8105 from GBrowseMaster group
+# (nothing else)
use strict;
use VM::EC2;
use VM::EC2::Instance::Metadata;
use Getopt::Long;
+$SIG{TERM} = \&terminate_instances;
+$SIG{INT} = \&terminate_instances;
+END { terminate_instances() }
+
# load averages:
# each item represents 15 min load average, lower and upper bound on instances
use constant LOAD_TABLE => [
@@ -33,16 +46,15 @@
'secret_key=s' => \$Secret_key,
) or exec 'perldoc',$0;
-my $extra_size = shift or die "Please provide size to grow /opt/gbrowse by. Use --help for details.";
-
#setup defaults
$ENV{EC2_ACCESS_KEY} = $Access_key if defined $Access_key;
$ENV{EC2_SECRET_KEY} = $Secret_key if defined $Secret_key;
-my $meta = VM::EC2::Instance::Metadata->new();
-my $imageId = $meta->imageId;
-my $zone = $meta->availabilityZone;
-my @groups = $meta->securityGroups;
+my $meta = VM::EC2::Instance::Metadata->new();
+my $imageId = $meta->imageId;
+my $instanceId = $meta->instanceId;
+my $zone = $meta->availabilityZone;
+my @groups = $meta->securityGroups;
(my $region = $zone) =~ s/[a-z]$//; # zone=>region
@@ -55,6 +67,8 @@
sleep (POLL_INTERVAL * 60);
}
+terminate_instances();
+
exit 0;
sub get_load {
@@ -90,6 +104,7 @@ sub adjust_spot_requests {
my $state = $sr->state;
my $instance = $sr->instance;
if ($state eq 'open' or ($instance && $instance->instanceState =~ /running|pending/)) {
+ $instance->add_tag(GBrowseMaster => $instanceId); # we'll use this to terminate all slaves sometime later
push @potential_instances,$instance || $state;
}
}
@@ -117,6 +132,7 @@ sub adjust_spot_requests {
-security_group => SECURITY_GROUP,
-user_data => "#!/bin/bash\n/etc/init.d/gbrowse-slave start\n",
);
+ $_->add_tag(Requestor=>'gbrowse_launch_aws_slaves') foreach @requests;
push @potential_instances,@requests;
}
return @potential_instances;
@@ -131,6 +147,22 @@ sub adjust_configuration {
my @a = map {("http://$_:8101",
"http://$_:8102",
"http://$_:8103")} @addresses;
- my @args = map {('--add'=> "$_") } @a;
+ my @args = map {('--set'=> "$_") } @a;
system 'sudo','gbrowse_add_slaves.pl',@args;
}
+
+sub terminate_instances {
+ $ec2 or return;
+ warn "terminating all slave instances";
+ my @spot_requests = $ec2->describe_spot_instance_requests({'tag:Requestor' => 'gbrowse_launch_aws_slaves'});
+ my @instances = $ec2->describe_instances({'tag:GBrowseMaster'=>$instanceId});
+ my %to_terminate = map {$_=>1} @instances;
+ foreach (@spot_requests) {
+ $to_terminate{$_->instance}++;
+ $ec2->cancel_spot_instance_requests($_);
+ }
+ $ec2->terminate_instances(keys %to_terminate);
+ system 'sudo','gbrowse_add_slaves.pl','--set','';
+}
+
+END { terminate_instances() }

0 comments on commit d2639db

Please sign in to comment.