Permalink
Browse files

Merge pull request #225 from nickmarden/refactor/prebuild_shared_reso…

…urces

Refactor/prebuild shared resources
  • Loading branch information...
2 parents 4a6dc23 + 2e6524d commit 19035016fbdfa072262d1bc06277cb4c82fc54a3 @temujin9 temujin9 committed Dec 14, 2012
@@ -75,6 +75,10 @@ def confirm_execution(target)
confirm_or_exit("Are you absolutely certain that you want to delete #{delete_message}? (Type 'Yes' to confirm) ", 'Yes')
end
+ def prepares?
+ false
+ end
+
end
end
end
@@ -82,6 +82,11 @@ def run
section("Syncing to chef")
target.save :providers => :chef
+ unless target.empty?
+ ui.info "Preparing shared resources:"
+ all_computers(*@name_args).prepare
+ end
+
# Launch computers
ui.info("")
section("Launching computers", :green)
@@ -119,6 +119,10 @@ def proxy_pac_contents
}\n}
end
+ def prepares?
+ false
+ end
+
def aggregates?
false
end
@@ -62,6 +62,10 @@ def perform_execution(target)
else Chef::Log.debug("Skipping sync to cloud") ; end
end
+ def prepares_on_noop?
+ true
+ end
+
def aggregates_on_noop?
true
end
@@ -44,6 +44,11 @@ def run
target = get_relevant_slice(* @name_args)
+ if prepares? and (prepares_on_noop? or not target.empty?)
+ ui.info "Preparing shared resources:"
+ all_computers(*@name_args).prepare
+ end
+
unless target.empty?
ui.info(["\n",
ui.color("Running #{sub_command}", :cyan),
@@ -76,6 +81,14 @@ def perform_execution(target)
target.send(sub_command)
end
+ def prepares?
+ true
+ end
+
+ def prepares_on_noop?
+ false
+ end
+
def aggregates?
true
end
@@ -298,11 +298,19 @@ def validate
values.map {|c| c.providers.values}.flatten.uniq.each {|p| p.validate computers }
end
- def aggregate
+ def group_action(verb)
computers = self
- provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas})}.flatten.uniq
+ provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas })}.flatten.uniq
providers = provider_keys.map { |pk| values.map { |c| c.providers[pk] } }.flatten.compact.uniq
- providers.each { |p| p.aggregate! computers }
+ providers.each { |p| p.send(verb, computers) }
+ end
+
+ def prepare
+ group_action(:prepare!)
+ end
+
+ def aggregate
+ group_action(:aggregate!)
end
#
View
@@ -42,6 +42,12 @@ def self.validate(computers)
resources.each {|r| r.validate_resources! computers }
end
+ def self.prepare!(computers)
+ resources.each do |r|
+ r.prepare!(computers) if r.shared?
+ end
+ end
+
def self.aggregate!(computers)
resources.each do |r|
r.aggregate!(computers) if r.shared?
@@ -87,14 +93,15 @@ def on_correlate(*p) Ironfan.noop(self,__method__,*p); end
#
def self.create!(*p) Ironfan.noop(self,__method__,*p); end
def self.save!(*p) Ironfan.noop(self,__method__,*p); end
+ def self.prepare!(*p) Ironfan.noop(self,__method__,*p); end
def self.aggregate!(*p) Ironfan.noop(self,__method__,*p); end
def self.destroy!(*p) Ironfan.noop(self,__method__,*p); end
#
# Utilities
#
- [:shared?, :multiple?, :load!,:validate_computer!,
- :validate_resources!,:create!,:save!,:aggregate!,:destroy!].each do |method_name|
+ [:shared?, :multiple?, :load!,:validate_computer!, :validate_resources!,
+ :create!, :save!, :prepare!, :aggregate!, :destroy!].each do |method_name|
define_method(method_name) {|*p| self.class.send(method_name,*p) }
end
@@ -50,8 +50,9 @@ def receive_adaptee(obj)
# Manipulation
#
- def self.create!(computer)
- name = computer.server.cluster_name
+ def self.prepare!(computers)
+ return if computers.empty?
+ name = computers.values[0].server.cluster_name
return if recall? name
Ironfan.step(name, "creating key pair for #{name}", :blue)
result = Ec2.connection.create_key_pair(name)
@@ -20,22 +20,18 @@ def self.multiple?() true; end
def self.resource_type() :security_group; end
def self.expected_ids(computer)
ec2 = computer.server.cloud(:ec2)
- ec2.security_groups.keys.map do |name|
- ec2.vpc ? "#{ec2.vpc}:#{name.to_s}" : name.to_s
- end.uniq
+ ec2.security_groups.keys.map { |name| group_name_with_vpc(name,ec2.vpc) }.uniq
end
def name()
- return adaptee.name if adaptee.vpc_id.nil?
- "#{adaptee.vpc_id}:#{adaptee.name}"
+ self.class.group_name_with_vpc(adaptee.name, adaptee.vpc_id)
end
#
# Discovery
#
def self.load!(cluster=nil)
- Ec2.connection.security_groups.each do |raw|
- next if raw.blank?
+ Ec2.connection.security_groups.reject { |raw| raw.blank? }.each do |raw|
sg = SecurityGroup.new(:adaptee => raw)
remember(sg)
Chef::Log.debug("Loaded #{sg}: #{sg.inspect}")
@@ -66,75 +62,119 @@ def to_s
# Manipulation
#
- def self.create!(computer)
- return unless Ec2.applicable computer
+ def self.prepare!(computers)
+
+ # Create any groups that don't yet exist, and ensure any authorizations
+ # that are required for those groups
+ cluster_name = nil
+ groups_to_create = [ ]
+ authorizations_to_ensure = [ ]
+
+ # First, deduce the list of all groups to which at least one instance belongs
+ # We'll use this later to decide whether to create groups, or authorize access,
+ # using a VPC security group or an EC2 security group.
+ groups_that_should_exist = computers.map { |c| expected_ids(c) }.flatten.sort.uniq
+ groups_to_create << groups_that_should_exist
+
+ computers.select { |computer| Ec2.applicable computer }.each do |computer|
+ ensure_groups(computer) # Add facet and cluster security groups for the computer
+ cloud = computer.server.cloud(:ec2)
+ cluster_name = computer.server.cluster_name
+
+ # Iterate over all of the security group information, keeping track of
+ # any groups that must exist and any authorizations that must be ensured
+ cloud.security_groups.values.each do |dsl_group|
+
+ groups_to_create << dsl_group.name
+
+ groups_to_create << dsl_group.group_authorized.map do |other_group|
+ most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist)
+ end
+
+ groups_to_create << dsl_group.group_authorized_by.map do |other_group|
+ most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist)
+ end
+
+ authorizations_to_ensure << dsl_group.group_authorized.map do |other_group|
+ {
+ :grantor => most_appropriate_group_name(dsl_group.name, cloud.vpc, groups_that_should_exist),
+ :grantee => most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist),
+ :grantee_type => :group,
+ :range => WIDE_OPEN,
+ }
+ end
+
+ authorizations_to_ensure << dsl_group.group_authorized_by.map do |other_group|
+ {
+ :grantor => most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist),
+ :grantee => most_appropriate_group_name(dsl_group.name, cloud.vpc, groups_that_should_exist),
+ :grantee_type => :group,
+ :range => WIDE_OPEN,
+ }
+ end
+
+ authorizations_to_ensure << dsl_group.range_authorizations.map do |range_auth|
+ range, cidr, protocol = range_auth
+ {
+ :grantor => group_name_with_vpc(dsl_group.name, cloud.vpc),
+ :grantee => { :cidr_ip => cidr, :ip_protocol => protocol },
+ :grantee_type => :cidr,
+ :range => range,
+ }
+ end
+ end
+ end
+ groups_to_create = groups_to_create.flatten.uniq.reject { |group| recall? group.to_s }.sort
+ authorizations_to_ensure = authorizations_to_ensure.flatten.uniq.sort { |a,b| a[:grantor] <=> b[:grantor] }
+
+ Ironfan.step(cluster_name, "creating security groups", :blue) unless groups_to_create.empty?
+ groups_to_create.each do |group|
+ if group =~ /\//
+ Ironfan.step(group, " assuming that owner/group pair #{group} already exists", :blue)
+ else
+ Ironfan.step(group, " creating #{group} security group", :blue)
+ begin
+ tokens = group.to_s.split(':')
+ group_id = tokens.pop
+ vpc_id = tokens.pop
+ Ec2.connection.create_security_group(group_id,"Ironfan created group #{group_id}",vpc_id)
+ rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
+ Chef::Log.info("ignoring security group error: #{e}")
+ end
+ end
+ end
- ensure_groups(computer)
- groups = self.expected_ids(computer)
- # Only handle groups that don't already exist
- groups.delete_if {|group| recall? group.to_s }
- return if groups.empty?
-
- Ironfan.step(computer.server.cluster_name, "creating security groups", :blue)
- groups.each do |group|
- Ironfan.step(group, " creating #{group} security group", :blue)
- begin
- tokens = group.to_s.split(':')
- group_id = tokens.pop
- vpc_id = tokens.pop
- Ec2.connection.create_security_group(group_id,"Ironfan created group #{group_id}",vpc_id)
- rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
- Chef::Log.info("ignoring security group error: #{e}")
- sleep 0.5 # quit racing so hard
+ # Re-load everything so that we have a @@known list of security groups to manipulate
+ load! unless groups_to_create.empty?
+
+ # Now make sure that all required authorizations are present
+ Ironfan.step(cluster_name, "ensuring security group permissions", :blue) unless authorizations_to_ensure.empty?
+ authorizations_to_ensure.each do |auth|
+ grantor_fog = recall(auth[:grantor])
+ if :group == auth[:grantee_type]
+ if fog_grantee = recall(auth[:grantee])
+ options = { :group => fog_grantee.group_id }
+ elsif auth[:grantee] =~ /\//
+ options = { :group_alias => auth[:grantee] }
+ else
+ raise "Don't know what to do with authorization grantee #{auth[:grantee]}"
+ end
+ message = " ensuring access from #{auth[:grantee]} to #{auth[:grantor]}"
+ else
+ options = auth[:grantee]
+ message = " ensuring #{auth[:grantee][:ip_protocol]} access from #{auth[:grantee][:cidr_ip]} to #{auth[:range]}"
end
+ Ironfan.step(auth[:grantor], message, :blue)
+ safely_authorize(grantor_fog, auth[:range], options)
end
- load! # Get the native groups via reload
end
- def self.recall_with_vpc(name,vpc_id=nil)
- group_name = vpc_id.nil? ? name : "#{vpc_id}:#{name}"
- recall(group_name)
+ def self.group_name_with_vpc(name,vpc_id=nil)
+ vpc_id.nil? ? name.to_s : "#{vpc_id}:#{name.to_s}"
end
- def self.save!(computer)
- return unless Ec2.applicable computer
- cloud = computer.server.cloud(:ec2)
-
- create!(computer) # Make sure the security groups exist
- security_groups = cloud.security_groups.values
- dsl_groups = security_groups.select do |dsl_group|
- not (recall_with_vpc(dsl_group,cloud.vpc)) and \
- not (dsl_group.range_authorizations +
- dsl_group.group_authorized_by +
- dsl_group.group_authorized).empty?
- end.compact
- return if dsl_groups.empty?
-
- Ironfan.step(computer.server.cluster_name, "ensuring security group permissions", :blue)
- dsl_groups.each do |dsl_group|
- dsl_group_fog = recall_with_vpc(dsl_group.name,cloud.vpc)
- dsl_group.group_authorized.each do |other_group|
- other_group_fog = recall_with_vpc(other_group,cloud.vpc)
- Ironfan.step(dsl_group.name, " ensuring access from #{other_group}", :blue)
- options = {:group => other_group_fog.group_id}
- safely_authorize(dsl_group_fog, WIDE_OPEN, options)
- end
-
- dsl_group.group_authorized_by.each do |other_group|
- other_group_fog = recall_with_vpc(other_group,cloud.vpc)
- Ironfan.step(dsl_group.name, " ensuring access to #{other_group}", :blue)
- options = {:group => dsl_group_fog.group_id}
- safely_authorize(other_group_fog, WIDE_OPEN, options)
- end
-
- dsl_group.range_authorizations.each do |range_auth|
- range, cidr, protocol = range_auth
- step_message = " ensuring #{protocol} access from #{cidr} to #{range}"
- Ironfan.step(dsl_group.name, step_message, :blue)
- options = {:cidr_ip => cidr, :ip_protocol => protocol}
- safely_authorize(dsl_group_fog, range, options)
- end
- end
+ def self.most_appropriate_group_name(group, vpc_id, all_valid_groups)
+ all_valid_groups.include?(group_name_with_vpc(group, vpc_id)) ? group_name_with_vpc(group, vpc_id) : group
end
#
@@ -156,20 +196,27 @@ def self.ensure_groups(computer)
# Try an authorization, ignoring duplicates (this is easier than correlating).
# Do so for both TCP and UDP, unless only one is specified
def self.safely_authorize(fog_group,range,options)
- unless options[:ip_protocol]
+ if options[:group_alias]
+ owner, group = options[:group_alias].split(/\//)
+ self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
+ Ec2.connection.authorize_security_group_ingress(
+ 'GroupName' => fog_group.name,
+ 'SourceSecurityGroupName' => group,
+ 'SourceSecurityGroupOwnerId' => owner
+ )
+ end
+ elsif options[:ip_protocol]
+ self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
+ fog_group.authorize_port_range(range,options)
+ end
+ else
safely_authorize(fog_group,range,options.merge(:ip_protocol => 'tcp'))
safely_authorize(fog_group,range,options.merge(:ip_protocol => 'udp'))
safely_authorize(fog_group,Range.new(-1,-1),options.merge(:ip_protocol => 'icmp')) if(range == WIDE_OPEN)
return
end
-
- self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
- fog_group.authorize_port_range(range,options)
- end
-
end
end
-
end
end
end
Oops, something went wrong.

0 comments on commit 1903501

Please sign in to comment.