Merge branch 'master' of https://github.com/infochimps-labs/ironfan i…

…nto feature/Elastic_IP_enhanced

CHANGELOG.md

@@ -1,3 +1,17 @@
+# v4.7.1
+* Cleaning up omnibus usage to link embedded bin, ruby into default $PATHs, rather than use /etc/environment to try tweaking (doesn't hit a large number of programs)
+* Launched machines should announce their state as "started"
+
+# v4.7.0:
+(@nickmarden rocks the house again)
+* Added support for "prepare" phase, prior to any machine-specific actions
+* Move security group creation and authorization assurance to prepare phase (fixes #189)
+* Allow user/group-style security group references (fixes #207)
+* Move keypair creation to prepare phase
+
+# v4.6.2:
+* Added a -f/--with-facet option to knife cluster list
+
 # v4.6.1:
 * Fixes nested array bug when computing list of AZs for an ELB (thanks @nickmarden)
 * Cleaning up overzealous Elastic IP inclusion (alternative fix to #222, thanks @nickmarden)

VERSION

@@ -1 +1 @@
-4.6.1
+4.7.1

ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.6.1"
+  s.version = "4.7.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-12-11"
+  s.date = "2012-12-20"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [

lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb

@@ -44,15 +44,12 @@ if [ ! -f /opt/chef/bin/chef-client ]; then
   curl -L http://www.opscode.com/chef/install.sh | sudo bash
 fi
 
-# Include the omnibus' path in the system-wide path, to allow use of
-#   its executables (gem, ruby, bundler, etc.)
-(
-cat <<'EOP'
-PATH="/opt/chef/embedded/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games"
-EOP
-) >> /etc/environment
-. /etc/environment
+# Replace /usr/local/sbin with the omnibus' bin, to get it into the PATH
+mv /usr/local/sbin{,~}
+ln -s /opt/chef/embedded/bin /usr/local/sbin
 
+# Link ruby into /usr/bin/ruby, to allow /usr/bin/env 
+ln -s /opt/chef/embedded/bin/ruby /usr/bin
 
 gem install extlib bundler json right_aws pry fog
 

lib/chef/knife/cluster_kill.rb

@@ -75,6 +75,10 @@ def confirm_execution(target)
         confirm_or_exit("Are you absolutely certain that you want to delete #{delete_message}? (Type 'Yes' to confirm) ", 'Yes')
       end
 
+      def prepares?
+        false
+      end
+
     end
   end
 end

lib/chef/knife/cluster_launch.rb

@@ -82,6 +82,11 @@ def run
         section("Syncing to chef")
         target.save :providers => :chef
 
+        unless target.empty?
+          ui.info "Preparing shared resources:"
+          all_computers(*@name_args).prepare
+        end
+
         # Launch computers
         ui.info("")
         section("Launching computers", :green)

lib/chef/knife/cluster_list.rb

@@ -27,23 +27,31 @@ class ClusterList < Knife
         require 'formatador'
       end
 
-      banner "knife cluster list (options)"
-
+      banner 'knife cluster list (options)'
+
+      option :facets,
+        :long        => '--with-facets',
+        :short       => '-f',
+        :description => 'List cluster facets along with names and paths',
+        :default     => false,
+        :boolean     => true
+
       def run
         load_ironfan
         configure_dry_run
 
-        hash = Ironfan.cluster_filenames
-
-        table = []
-        hash.keys.sort.each do |key|
-          table.push( { :cluster => key, :path => hash[key] } )
+        data = Ironfan.cluster_filenames.map do |name, path|
+          as_table = { :cluster => name, :path => path }
+          if config[:facets]
+            facets = Ironfan.load_cluster(name).facets.to_a.map(&:name).join(', ')
+            as_table.merge!(:facets => facets)
+          end
+          as_table
         end
 
         ui.info "Cluster Path: #{ Ironfan.cluster_path.join ", " }"
-
-        Formatador.display_compact_table(table, [:cluster,:path])
-
+        headers = config[:facets] ? [:cluster, :facets, :path] : [:cluster, :path] 
+        Formatador.display_compact_table(data, headers)
       end
     end
   end

lib/chef/knife/cluster_proxy.rb

@@ -119,6 +119,10 @@ def proxy_pac_contents
 }\n}
       end
 
+      def prepares?
+        false
+      end
+
       def aggregates?
         false
       end

lib/chef/knife/cluster_sync.rb

@@ -62,6 +62,10 @@ def perform_execution(target)
         else Chef::Log.debug("Skipping sync to cloud") ; end
       end
 
+      def prepares_on_noop?
+        true
+      end
+
       def aggregates_on_noop?
         true
       end

lib/chef/knife/ironfan_script.rb

@@ -44,6 +44,11 @@ def run
 
       target = get_relevant_slice(* @name_args)
 
+      if prepares? and (prepares_on_noop? or not target.empty?)
+        ui.info "Preparing shared resources:"
+        all_computers(*@name_args).prepare
+      end
+
       unless target.empty?
         ui.info(["\n",
                  ui.color("Running #{sub_command}", :cyan),
@@ -76,6 +81,14 @@ def perform_execution(target)
       target.send(sub_command)
     end
 
+    def prepares?
+      true
+    end
+
+    def prepares_on_noop?
+      false
+    end
+
     def aggregates?
       true
     end

lib/ironfan/broker/computer.rb

@@ -75,6 +75,7 @@ def kill(options={})
       def launch
         ensure_dependencies
         iaas_provider.machine_class.create! self
+        node.announce_state :started
         save
         self
       end
@@ -298,11 +299,19 @@ def validate
         values.map {|c| c.providers.values}.flatten.uniq.each {|p| p.validate computers }
       end
 
-      def aggregate
+      def group_action(verb)
         computers = self
-        provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas})}.flatten.uniq
+        provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas })}.flatten.uniq
         providers     = provider_keys.map { |pk| values.map { |c| c.providers[pk] } }.flatten.compact.uniq
-        providers.each { |p| p.aggregate! computers }
+        providers.each { |p| p.send(verb, computers) }
+      end
+
+      def prepare
+        group_action(:prepare!)
+      end
+
+      def aggregate
+        group_action(:aggregate!)
       end
 
       #

lib/ironfan/provider.rb

@@ -42,6 +42,12 @@ def self.validate(computers)
       resources.each {|r| r.validate_resources! computers }
     end
 
+    def self.prepare!(computers)
+      resources.each do |r|
+        r.prepare!(computers) if r.shared?
+      end
+    end
+
     def self.aggregate!(computers)
       resources.each do |r|
         r.aggregate!(computers) if r.shared?
@@ -87,14 +93,15 @@ def on_correlate(*p)              Ironfan.noop(self,__method__,*p);   end
       #
       def self.create!(*p)              Ironfan.noop(self,__method__,*p);   end
       def self.save!(*p)                Ironfan.noop(self,__method__,*p);   end
+      def self.prepare!(*p)             Ironfan.noop(self,__method__,*p);   end
       def self.aggregate!(*p)           Ironfan.noop(self,__method__,*p);   end
       def self.destroy!(*p)             Ironfan.noop(self,__method__,*p);   end
 
       #
       # Utilities
       #
-      [:shared?, :multiple?, :load!,:validate_computer!,
-       :validate_resources!,:create!,:save!,:aggregate!,:destroy!].each do |method_name|
+      [:shared?, :multiple?, :load!,:validate_computer!, :validate_resources!,
+       :create!, :save!, :prepare!, :aggregate!, :destroy!].each do |method_name|
          define_method(method_name) {|*p| self.class.send(method_name,*p) }
        end
 

lib/ironfan/provider/ec2/keypair.rb

@@ -50,8 +50,9 @@ def receive_adaptee(obj)
         # Manipulation
         #
 
-        def self.create!(computer)
-          name = computer.server.cluster_name
+        def self.prepare!(computers)
+          return if computers.empty?
+          name = computers.values[0].server.cluster_name
           return if recall? name
           Ironfan.step(name, "creating key pair for #{name}", :blue)
           result = Ec2.connection.create_key_pair(name)