Skip to content
Browse files

Merge branch 'master' of https://github.com/infochimps-labs/ironfan i…

…nto feature/Elastic_IP_enhanced
  • Loading branch information...
2 parents 8fef5c5 + 1a84a44 commit 8188f35d4834dee443afdb6e3d95a6812fd19dc3 Jerry Jackson committed Dec 20, 2012
View
14 CHANGELOG.md
@@ -1,3 +1,17 @@
+# v4.7.1
+* Cleaning up omnibus usage to link embedded bin, ruby into default $PATHs, rather than use /etc/environment to try tweaking (doesn't hit a large number of programs)
+* Launched machines should announce their state as "started"
+
+# v4.7.0:
+(@nickmarden rocks the house again)
+* Added support for "prepare" phase, prior to any machine-specific actions
+* Move security group creation and authorization assurance to prepare phase (fixes #189)
+* Allow user/group-style security group references (fixes #207)
+* Move keypair creation to prepare phase
+
+# v4.6.2:
+* Added a -f/--with-facet option to knife cluster list
+
# v4.6.1:
* Fixes nested array bug when computing list of AZs for an ELB (thanks @nickmarden)
* Cleaning up overzealous Elastic IP inclusion (alternative fix to #222, thanks @nickmarden)
View
2 VERSION
@@ -1 +1 @@
-4.6.1
+4.7.1
View
4 ironfan.gemspec
@@ -5,11 +5,11 @@
Gem::Specification.new do |s|
s.name = "ironfan"
- s.version = "4.6.1"
+ s.version = "4.7.1"
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Infochimps"]
- s.date = "2012-12-11"
+ s.date = "2012-12-20"
s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
s.email = "coders@infochimps.com"
s.extra_rdoc_files = [
View
13 lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb
@@ -44,15 +44,12 @@ if [ ! -f /opt/chef/bin/chef-client ]; then
curl -L http://www.opscode.com/chef/install.sh | sudo bash
fi
-# Include the omnibus' path in the system-wide path, to allow use of
-# its executables (gem, ruby, bundler, etc.)
-(
-cat <<'EOP'
-PATH="/opt/chef/embedded/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games"
-EOP
-) >> /etc/environment
-. /etc/environment
+# Replace /usr/local/sbin with the omnibus' bin, to get it into the PATH
+mv /usr/local/sbin{,~}
+ln -s /opt/chef/embedded/bin /usr/local/sbin
+# Link ruby into /usr/bin/ruby, to allow /usr/bin/env
+ln -s /opt/chef/embedded/bin/ruby /usr/bin
gem install extlib bundler json right_aws pry fog
View
4 lib/chef/knife/cluster_kill.rb
@@ -75,6 +75,10 @@ def confirm_execution(target)
confirm_or_exit("Are you absolutely certain that you want to delete #{delete_message}? (Type 'Yes' to confirm) ", 'Yes')
end
+ def prepares?
+ false
+ end
+
end
end
end
View
5 lib/chef/knife/cluster_launch.rb
@@ -82,6 +82,11 @@ def run
section("Syncing to chef")
target.save :providers => :chef
+ unless target.empty?
+ ui.info "Preparing shared resources:"
+ all_computers(*@name_args).prepare
+ end
+
# Launch computers
ui.info("")
section("Launching computers", :green)
View
28 lib/chef/knife/cluster_list.rb
@@ -27,23 +27,31 @@ class ClusterList < Knife
require 'formatador'
end
- banner "knife cluster list (options)"
-
+ banner 'knife cluster list (options)'
+
+ option :facets,
+ :long => '--with-facets',
+ :short => '-f',
+ :description => 'List cluster facets along with names and paths',
+ :default => false,
+ :boolean => true
+
def run
load_ironfan
configure_dry_run
- hash = Ironfan.cluster_filenames
-
- table = []
- hash.keys.sort.each do |key|
- table.push( { :cluster => key, :path => hash[key] } )
+ data = Ironfan.cluster_filenames.map do |name, path|
+ as_table = { :cluster => name, :path => path }
+ if config[:facets]
+ facets = Ironfan.load_cluster(name).facets.to_a.map(&:name).join(', ')
+ as_table.merge!(:facets => facets)
+ end
+ as_table
end
ui.info "Cluster Path: #{ Ironfan.cluster_path.join ", " }"
-
- Formatador.display_compact_table(table, [:cluster,:path])
-
+ headers = config[:facets] ? [:cluster, :facets, :path] : [:cluster, :path]
+ Formatador.display_compact_table(data, headers)
end
end
end
View
4 lib/chef/knife/cluster_proxy.rb
@@ -119,6 +119,10 @@ def proxy_pac_contents
}\n}
end
+ def prepares?
+ false
+ end
+
def aggregates?
false
end
View
4 lib/chef/knife/cluster_sync.rb
@@ -62,6 +62,10 @@ def perform_execution(target)
else Chef::Log.debug("Skipping sync to cloud") ; end
end
+ def prepares_on_noop?
+ true
+ end
+
def aggregates_on_noop?
true
end
View
13 lib/chef/knife/ironfan_script.rb
@@ -44,6 +44,11 @@ def run
target = get_relevant_slice(* @name_args)
+ if prepares? and (prepares_on_noop? or not target.empty?)
+ ui.info "Preparing shared resources:"
+ all_computers(*@name_args).prepare
+ end
+
unless target.empty?
ui.info(["\n",
ui.color("Running #{sub_command}", :cyan),
@@ -76,6 +81,14 @@ def perform_execution(target)
target.send(sub_command)
end
+ def prepares?
+ true
+ end
+
+ def prepares_on_noop?
+ false
+ end
+
def aggregates?
true
end
View
15 lib/ironfan/broker/computer.rb
@@ -75,6 +75,7 @@ def kill(options={})
def launch
ensure_dependencies
iaas_provider.machine_class.create! self
+ node.announce_state :started
save
self
end
@@ -298,11 +299,19 @@ def validate
values.map {|c| c.providers.values}.flatten.uniq.each {|p| p.validate computers }
end
- def aggregate
+ def group_action(verb)
computers = self
- provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas})}.flatten.uniq
+ provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas })}.flatten.uniq
providers = provider_keys.map { |pk| values.map { |c| c.providers[pk] } }.flatten.compact.uniq
- providers.each { |p| p.aggregate! computers }
+ providers.each { |p| p.send(verb, computers) }
+ end
+
+ def prepare
+ group_action(:prepare!)
+ end
+
+ def aggregate
+ group_action(:aggregate!)
end
#
View
11 lib/ironfan/provider.rb
@@ -42,6 +42,12 @@ def self.validate(computers)
resources.each {|r| r.validate_resources! computers }
end
+ def self.prepare!(computers)
+ resources.each do |r|
+ r.prepare!(computers) if r.shared?
+ end
+ end
+
def self.aggregate!(computers)
resources.each do |r|
r.aggregate!(computers) if r.shared?
@@ -87,14 +93,15 @@ def on_correlate(*p) Ironfan.noop(self,__method__,*p); end
#
def self.create!(*p) Ironfan.noop(self,__method__,*p); end
def self.save!(*p) Ironfan.noop(self,__method__,*p); end
+ def self.prepare!(*p) Ironfan.noop(self,__method__,*p); end
def self.aggregate!(*p) Ironfan.noop(self,__method__,*p); end
def self.destroy!(*p) Ironfan.noop(self,__method__,*p); end
#
# Utilities
#
- [:shared?, :multiple?, :load!,:validate_computer!,
- :validate_resources!,:create!,:save!,:aggregate!,:destroy!].each do |method_name|
+ [:shared?, :multiple?, :load!,:validate_computer!, :validate_resources!,
+ :create!, :save!, :prepare!, :aggregate!, :destroy!].each do |method_name|
define_method(method_name) {|*p| self.class.send(method_name,*p) }
end
View
5 lib/ironfan/provider/ec2/keypair.rb
@@ -50,8 +50,9 @@ def receive_adaptee(obj)
# Manipulation
#
- def self.create!(computer)
- name = computer.server.cluster_name
+ def self.prepare!(computers)
+ return if computers.empty?
+ name = computers.values[0].server.cluster_name
return if recall? name
Ironfan.step(name, "creating key pair for #{name}", :blue)
result = Ec2.connection.create_key_pair(name)
View
199 lib/ironfan/provider/ec2/security_group.rb
@@ -20,22 +20,18 @@ def self.multiple?() true; end
def self.resource_type() :security_group; end
def self.expected_ids(computer)
ec2 = computer.server.cloud(:ec2)
- ec2.security_groups.keys.map do |name|
- ec2.vpc ? "#{ec2.vpc}:#{name.to_s}" : name.to_s
- end.uniq
+ ec2.security_groups.keys.map { |name| group_name_with_vpc(name,ec2.vpc) }.uniq
end
def name()
- return adaptee.name if adaptee.vpc_id.nil?
- "#{adaptee.vpc_id}:#{adaptee.name}"
+ self.class.group_name_with_vpc(adaptee.name, adaptee.vpc_id)
end
#
# Discovery
#
def self.load!(cluster=nil)
- Ec2.connection.security_groups.each do |raw|
- next if raw.blank?
+ Ec2.connection.security_groups.reject { |raw| raw.blank? }.each do |raw|
sg = SecurityGroup.new(:adaptee => raw)
remember(sg)
Chef::Log.debug("Loaded #{sg}: #{sg.inspect}")
@@ -66,75 +62,119 @@ def to_s
# Manipulation
#
- def self.create!(computer)
- return unless Ec2.applicable computer
+ def self.prepare!(computers)
+
+ # Create any groups that don't yet exist, and ensure any authorizations
+ # that are required for those groups
+ cluster_name = nil
+ groups_to_create = [ ]
+ authorizations_to_ensure = [ ]
+
+ # First, deduce the list of all groups to which at least one instance belongs
+ # We'll use this later to decide whether to create groups, or authorize access,
+ # using a VPC security group or an EC2 security group.
+ groups_that_should_exist = computers.map { |c| expected_ids(c) }.flatten.sort.uniq
+ groups_to_create << groups_that_should_exist
+
+ computers.select { |computer| Ec2.applicable computer }.each do |computer|
+ ensure_groups(computer) # Add facet and cluster security groups for the computer
+ cloud = computer.server.cloud(:ec2)
+ cluster_name = computer.server.cluster_name
+
+ # Iterate over all of the security group information, keeping track of
+ # any groups that must exist and any authorizations that must be ensured
+ cloud.security_groups.values.each do |dsl_group|
+
+ groups_to_create << dsl_group.name
+
+ groups_to_create << dsl_group.group_authorized.map do |other_group|
+ most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist)
+ end
+
+ groups_to_create << dsl_group.group_authorized_by.map do |other_group|
+ most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist)
+ end
+
+ authorizations_to_ensure << dsl_group.group_authorized.map do |other_group|
+ {
+ :grantor => most_appropriate_group_name(dsl_group.name, cloud.vpc, groups_that_should_exist),
+ :grantee => most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist),
+ :grantee_type => :group,
+ :range => WIDE_OPEN,
+ }
+ end
+
+ authorizations_to_ensure << dsl_group.group_authorized_by.map do |other_group|
+ {
+ :grantor => most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist),
+ :grantee => most_appropriate_group_name(dsl_group.name, cloud.vpc, groups_that_should_exist),
+ :grantee_type => :group,
+ :range => WIDE_OPEN,
+ }
+ end
+
+ authorizations_to_ensure << dsl_group.range_authorizations.map do |range_auth|
+ range, cidr, protocol = range_auth
+ {
+ :grantor => group_name_with_vpc(dsl_group.name, cloud.vpc),
+ :grantee => { :cidr_ip => cidr, :ip_protocol => protocol },
+ :grantee_type => :cidr,
+ :range => range,
+ }
+ end
+ end
+ end
+ groups_to_create = groups_to_create.flatten.uniq.reject { |group| recall? group.to_s }.sort
+ authorizations_to_ensure = authorizations_to_ensure.flatten.uniq.sort { |a,b| a[:grantor] <=> b[:grantor] }
+
+ Ironfan.step(cluster_name, "creating security groups", :blue) unless groups_to_create.empty?
+ groups_to_create.each do |group|
+ if group =~ /\//
+ Ironfan.step(group, " assuming that owner/group pair #{group} already exists", :blue)
+ else
+ Ironfan.step(group, " creating #{group} security group", :blue)
+ begin
+ tokens = group.to_s.split(':')
+ group_id = tokens.pop
+ vpc_id = tokens.pop
+ Ec2.connection.create_security_group(group_id,"Ironfan created group #{group_id}",vpc_id)
+ rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
+ Chef::Log.info("ignoring security group error: #{e}")
+ end
+ end
+ end
- ensure_groups(computer)
- groups = self.expected_ids(computer)
- # Only handle groups that don't already exist
- groups.delete_if {|group| recall? group.to_s }
- return if groups.empty?
-
- Ironfan.step(computer.server.cluster_name, "creating security groups", :blue)
- groups.each do |group|
- Ironfan.step(group, " creating #{group} security group", :blue)
- begin
- tokens = group.to_s.split(':')
- group_id = tokens.pop
- vpc_id = tokens.pop
- Ec2.connection.create_security_group(group_id,"Ironfan created group #{group_id}",vpc_id)
- rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
- Chef::Log.info("ignoring security group error: #{e}")
- sleep 0.5 # quit racing so hard
+ # Re-load everything so that we have a @@known list of security groups to manipulate
+ load! unless groups_to_create.empty?
+
+ # Now make sure that all required authorizations are present
+ Ironfan.step(cluster_name, "ensuring security group permissions", :blue) unless authorizations_to_ensure.empty?
+ authorizations_to_ensure.each do |auth|
+ grantor_fog = recall(auth[:grantor])
+ if :group == auth[:grantee_type]
+ if fog_grantee = recall(auth[:grantee])
+ options = { :group => fog_grantee.group_id }
+ elsif auth[:grantee] =~ /\//
+ options = { :group_alias => auth[:grantee] }
+ else
+ raise "Don't know what to do with authorization grantee #{auth[:grantee]}"
+ end
+ message = " ensuring access from #{auth[:grantee]} to #{auth[:grantor]}"
+ else
+ options = auth[:grantee]
+ message = " ensuring #{auth[:grantee][:ip_protocol]} access from #{auth[:grantee][:cidr_ip]} to #{auth[:range]}"
end
+ Ironfan.step(auth[:grantor], message, :blue)
+ safely_authorize(grantor_fog, auth[:range], options)
end
- load! # Get the native groups via reload
end
- def self.recall_with_vpc(name,vpc_id=nil)
- group_name = vpc_id.nil? ? name : "#{vpc_id}:#{name}"
- recall(group_name)
+ def self.group_name_with_vpc(name,vpc_id=nil)
+ vpc_id.nil? ? name.to_s : "#{vpc_id}:#{name.to_s}"
end
- def self.save!(computer)
- return unless Ec2.applicable computer
- cloud = computer.server.cloud(:ec2)
-
- create!(computer) # Make sure the security groups exist
- security_groups = cloud.security_groups.values
- dsl_groups = security_groups.select do |dsl_group|
- not (recall_with_vpc(dsl_group,cloud.vpc)) and \
- not (dsl_group.range_authorizations +
- dsl_group.group_authorized_by +
- dsl_group.group_authorized).empty?
- end.compact
- return if dsl_groups.empty?
-
- Ironfan.step(computer.server.cluster_name, "ensuring security group permissions", :blue)
- dsl_groups.each do |dsl_group|
- dsl_group_fog = recall_with_vpc(dsl_group.name,cloud.vpc)
- dsl_group.group_authorized.each do |other_group|
- other_group_fog = recall_with_vpc(other_group,cloud.vpc)
- Ironfan.step(dsl_group.name, " ensuring access from #{other_group}", :blue)
- options = {:group => other_group_fog.group_id}
- safely_authorize(dsl_group_fog, WIDE_OPEN, options)
- end
-
- dsl_group.group_authorized_by.each do |other_group|
- other_group_fog = recall_with_vpc(other_group,cloud.vpc)
- Ironfan.step(dsl_group.name, " ensuring access to #{other_group}", :blue)
- options = {:group => dsl_group_fog.group_id}
- safely_authorize(other_group_fog, WIDE_OPEN, options)
- end
-
- dsl_group.range_authorizations.each do |range_auth|
- range, cidr, protocol = range_auth
- step_message = " ensuring #{protocol} access from #{cidr} to #{range}"
- Ironfan.step(dsl_group.name, step_message, :blue)
- options = {:cidr_ip => cidr, :ip_protocol => protocol}
- safely_authorize(dsl_group_fog, range, options)
- end
- end
+ def self.most_appropriate_group_name(group, vpc_id, all_valid_groups)
+ all_valid_groups.include?(group_name_with_vpc(group, vpc_id)) ? group_name_with_vpc(group, vpc_id) : group
end
#
@@ -156,20 +196,27 @@ def self.ensure_groups(computer)
# Try an authorization, ignoring duplicates (this is easier than correlating).
# Do so for both TCP and UDP, unless only one is specified
def self.safely_authorize(fog_group,range,options)
- unless options[:ip_protocol]
+ if options[:group_alias]
+ owner, group = options[:group_alias].split(/\//)
+ self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
+ Ec2.connection.authorize_security_group_ingress(
+ 'GroupName' => fog_group.name,
+ 'SourceSecurityGroupName' => group,
+ 'SourceSecurityGroupOwnerId' => owner
+ )
+ end
+ elsif options[:ip_protocol]
+ self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
+ fog_group.authorize_port_range(range,options)
+ end
+ else
safely_authorize(fog_group,range,options.merge(:ip_protocol => 'tcp'))
safely_authorize(fog_group,range,options.merge(:ip_protocol => 'udp'))
safely_authorize(fog_group,Range.new(-1,-1),options.merge(:ip_protocol => 'icmp')) if(range == WIDE_OPEN)
return
end
-
- self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
- fog_group.authorize_port_range(range,options)
- end
-
end
end
-
end
end
end
View
22 spec/integration/spec/simple_cluster_spec.rb
@@ -17,6 +17,10 @@
facet :web do
instances 1
+ cloud(:ec2).security_group(:web) do
+ authorize_group :web_clients
+ authorize_group 'amazon-elb/amazon-elb-sg'
+ end
end
facet :db do
@@ -35,14 +39,20 @@
describe "the web facet security groups" do
subject { cluster.facets[:web].server(0).cloud(:ec2).security_groups.keys.map(&:to_s).sort }
- it { should == %w[ simple simple-web ssh systemwide ] }
+ it { should == %w[ simple simple-web ssh systemwide web ] }
end
describe "the db facet security groups" do
subject { cluster.facets[:db].server(0).cloud(:ec2).security_groups.keys.map(&:to_s).sort }
it { should == %w[ simple simple-db ssh systemwide ] }
end
+ describe "the passively created security groups" do
+ it "should include the :web_clients group" do
+ Ironfan::Provider::Ec2::SecurityGroup.recall('web_clients').should_not be_nil
+ end
+ end
+
describe "the cluster-wide security group" do
before :each do
@sg = Ironfan::Provider::Ec2::SecurityGroup.recall('simple')
@@ -76,7 +86,17 @@
@ordered_ipp['icmp']['fromPort'].to_i.should == -1
@ordered_ipp['icmp']['toPort'].to_i.should == -1
end
+ end
+
+ describe "the web security group" do
+ before :each do
+ @sg = Ironfan::Provider::Ec2::SecurityGroup.recall('web')
+ @ordered_ipp = Hash[ @sg.ip_permissions.map { |s| [ s['ipProtocol'], s ] } ]
+ end
+ it "allows TCP connections to web_clients and to amazon-elb-sg" do
+ @ordered_ipp['tcp']['groups'].map { |g| g['groupName'] }.sort.should == %w[ amazon-elb-sg web_clients ]
+ end
end
end
end

0 comments on commit 8188f35

Please sign in to comment.
Something went wrong with that request. Please try again.