Merge branch 'master' of github.com:infochimps-labs/ironfan into secu…

…rity_group_rescue

Conflicts:
	lib/ironfan/provider/chef/client.rb
	lib/ironfan/provider/chef/node.rb
	lib/ironfan/provider/chef/role.rb
	lib/ironfan/provider/ec2/ebs_volume.rb
	lib/ironfan/provider/ec2/key_pair.rb
	lib/ironfan/provider/ec2/machine.rb
	lib/ironfan/provider/ec2/placement_group.rb
	lib/ironfan/provider/ec2/security_group.rb

.rspec

@@ -1,3 +1,2 @@
 --color
---format documentation
---drb
+--format documentation

CHANGELOG.md

@@ -1,3 +1,17 @@
+# v4.2.1: @nickmarden rocks the house
+* More correct merging of cluster and facet objects, with specs
+* Circumvent memory bloat by resolving just once
+
+# v4.2.0:
+* providers now load in parallel
+* substeps announcements for load are now done in provider, not individual resources
+* only dump computers full information on -VV
+* more cleanup of new specs
+
+# v4.1.1: fixing 'rake spec'
+* Remove all defunct tests and start fresh (fixes #137)
+* Failing spec for #158 
+
 # v4.1.0: several bug-fixes and code cleanup
 * Splat the args to DSL::Volume.snapshot_id so we can call it properly (fixes #161)
 * cloud(:ec2) lets you declare bitness (fixes #147)

Rakefile

@@ -64,9 +64,7 @@ Jeweler::RubygemsDotOrgTasks.new
 #
 # RSpec -- testing
 #
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
-end
+RSpec::Core::RakeTask.new(:spec)
 
 RSpec::Core::RakeTask.new(:rcov) do |spec|
   spec.pattern = 'spec/**/*_spec.rb'

VERSION

@@ -1 +1 @@
-4.1.0
+4.2.1

ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.1.0"
+  s.version = "4.2.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-09-24"
+  s.date = "2012-10-01"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -85,10 +85,8 @@ Gem::Specification.new do |s|
     "lib/ironfan/provider/virtualbox/machine.rb",
     "lib/ironfan/requirements.rb",
     "spec/ironfan/cluster_spec.rb",
-    "spec/ironfan/facet_spec.rb",
-    "spec/ironfan/server_slice_spec.rb",
-    "spec/ironfan/server_spec.rb",
-    "spec/ironfan_spec.rb",
+    "spec/ironfan/ec2/cloud_provider_spec.rb",
+    "spec/ironfan/ec2/security_group_spec.rb",
     "spec/spec_helper.rb",
     "spec/spec_helper/dummy_chef.rb",
     "spec/test_config.rb",
@@ -99,7 +97,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = "1.8.24"
   s.summary = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
-  s.test_files = ["spec/ironfan_spec.rb", "spec/spec_helper/dummy_chef.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/server_slice_spec.rb", "spec/ironfan/server_spec.rb", "spec/ironfan/facet_spec.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
+  s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

lib/chef/knife/cluster_proxy.rb

@@ -48,7 +48,7 @@ class ClusterProxy < Ironfan::Script
         :default     => '6666'
 
       def relevant?(server)
-        server.sshable?
+        server.machine.running?
       end
 
       def perform_execution(target)
@@ -62,7 +62,7 @@ def perform_execution(target)
       def command_for_target(svr)
         config[:attribute]       ||= Chef::Config[:knife][:ssh_address_attribute] || "fqdn"
         config[:ssh_user]        ||= Chef::Config[:knife][:ssh_user]
-        config[:identity_file]   ||= svr.cloud.ssh_identity_file
+        config[:identity_file]   ||= svr.server.cloud.ssh_identity_file
         config[:host_key_verify] ||= Chef::Config[:knife][:host_key_verify] || (not config[:no_host_key_verify]) # pre-vs-post 0.10.4
 
         address = svr.public_hostname

lib/chef/knife/cluster_show.rb

@@ -35,6 +35,7 @@ class ClusterShow < Knife
         :boolean     => true
 
       def run
+        with_verbosity(1){ config[:include_terminated] }
         load_ironfan
         die(banner) if @name_args.empty?
         configure_dry_run
@@ -56,8 +57,12 @@ def run
     protected
 
       def dump_computer(computer)
+        header = "Computer #{computer.name} (#{computer.class})"
         with_verbosity 1 do
-          dump("Computer #{computer.name} (#{computer.class})", computer.to_wire)
+          Chef::Log.info(header)
+        end
+        with_verbosity 2 do
+          dump(header, computer.to_wire)
         end
       end
 

lib/gorillib/resolution.rb

@@ -19,6 +19,28 @@ module Model
     Field.class_eval do
       field :resolver, Symbol, :default => :read_set_or_underlay_attribute
     end
+
+    # Modifies the Gorillib metaprogramming to handle deep recursion on
+    # Gorillib::Model collections which would prefer to handle arbitrarily
+    # complex resolution requirements via their (custom) receive! method
+    module ClassMethods
+      def define_collection_receiver(field)
+       collection_field_name = field.name; collection_type = field.type
+        # @param  [Array[Object],Hash[Object]] the collection to merge
+        # @return [Gorillib::Collection] the updated collection
+        define_meta_module_method("receive_#{collection_field_name}", true) do |coll, &block|
+          begin
+            existing = read_attribute(collection_field_name)
+            if existing and (not collection_type.native?(coll) or existing.respond_to?(:receive!))
+              existing.receive!(coll, &block)
+            else
+              write_attribute(collection_field_name, coll)
+            end
+          rescue StandardError => err ; err.polish("#{self.class} #{collection_field_name} collection on #{args}'") rescue nil ; raise ; end
+        end
+      end
+    end
+
   end
 
   # The attribute :underlay provides an object (preferably another
@@ -50,6 +72,14 @@ def resolve
       result
     end
 
+    def resolve!
+      resolved = resolve
+      self.class.fields.each do |field_name, field|
+        write_attribute(field_name, resolved.send(field_name.to_sym))
+      end
+      self
+    end
+
     def deep_resolve(field_name)
       temp = read_set_or_underlay_attribute(field_name)
       return if temp.nil?
@@ -106,7 +136,7 @@ def read_set_attribute(field_name)
 
     def read_underlay_attribute(field_name)
       return if underlay.nil?
-      underlay.read_resolved_attribute(field_name)
+      Gorillib.deep_copy(underlay.read_resolved_attribute(field_name))
     end
 
     def read_set_or_underlay_attribute(field_name)

lib/ironfan.rb

@@ -15,7 +15,7 @@ def self.cluster_path
   #
   # Delegates
   def self.clusters
-    Chef::Config[:clusters] ||= Hash.new
+    @@clusters
   end
 
   def self.ui=(ui) @ui = ui ; end
@@ -61,9 +61,15 @@ def self.parallel(targets)
   #
   def self.cluster(name, attrs={}, &block)
     name = name.to_sym
-
-    cl = ( @@clusters[name] ||= Ironfan::Dsl::Cluster.new({:name => name}) )
-    cl.receive!(attrs, &block)
+    # If this is being called as Ironfan.cluster('foo') with no additional arguments,
+    # return the cached cluster object if it exists
+    if @@clusters[name] and attrs.empty? and not block_given?
+      return @@clusters[name]
+    else # Otherwise we're being asked to (re)initialize and cache a cluster definition
+      cl = Ironfan::Dsl::Cluster.new(:name => name)
+      cl.receive!(attrs, &block)
+      @@clusters[name] = cl.resolve
+    end
   end
 
   #
@@ -86,9 +92,6 @@ def self.load_cluster(cluster_name)
     require cluster_file
     unless @@clusters[cluster] then  die("#{cluster_file} was supposed to have the definition for the #{cluster_name} cluster, but didn't") end
 
-    # Flesh out the expected servers listed in the facets
-    @@clusters[cluster].expand_servers!
-
     @@clusters[cluster]
   end
 

lib/ironfan/broker.rb

@@ -13,10 +13,10 @@ class Broker < Builder
     #   unrecognizable resources are labeled as bogus.
     def discover!(cluster)
       # Get fully resolved servers, and build Computers using them
-      computers = Computers.new(:cluster => cluster.resolve)
+      computers = Computers.new(:cluster => cluster)
       #
       providers = computers.map{|c| c.providers.values }.flatten.uniq
-      providers.each do |provider|
+      Ironfan.parallel(providers) do |provider|
         Ironfan.step cluster.name, "Loading #{provider.handle}", :cyan
         provider.load cluster
       end

lib/ironfan/dsl/cluster.rb

@@ -7,12 +7,6 @@ class Cluster < Ironfan::Dsl::Compute
       def initialize(attrs={},&block)
         super
         self.cluster_role       Ironfan::Dsl::Role.new(:name => "#{attrs[:name]}_cluster")
-        self.expand_servers!
-      end
-
-      def expand_servers!
-        facets.each {|facet| facet.expand_servers! }
-        servers
       end
 
       # Utility method to reference all servers from constituent facets

lib/ironfan/dsl/facet.rb

@@ -11,14 +11,11 @@ def initialize(attrs={},&block)
         self.name               = attrs[:name] unless attrs[:name].nil?
         self.facet_role         Ironfan::Dsl::Role.new(:name => fullname.sub('-','_'))
         super
+        for i in 0 .. instances-1; server(i); end
       end
 
       def fullname()            "#{cluster_name}-#{name}";      end
 
-      def expand_servers!
-        for i in 0..(instances-1) do server(i); end
-      end
-
     end
 
   end

lib/ironfan/provider.rb

@@ -25,7 +25,12 @@ def self.resources
     # Discovery
     #
     def self.load(cluster)
-      resources.each {|r| r.load! cluster }
+      Ironfan.parallel (resources) do |r|
+        type = r.resource_type.to_s
+        Ironfan.substep(cluster.name, "loading #{type}s")
+        r.load! cluster
+        Ironfan.substep(cluster.name, "loaded #{type}s")
+      end
     end
 
     def self.validate(computers)

lib/ironfan/provider/chef/client.rb

@@ -45,9 +45,8 @@ def private_key(body=nil)
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ironfan.substep(cluster ? cluster.name : 'all', "chef clients")
-          nameq = "name:#{cluster.name}-* OR clientname:#{cluster.name}-*"
-          ChefServer.search(:client, nameq) do |raw|
+          query = cluster && "name:#{cluster.name}-* OR clientname:#{cluster.name}-*"
+          ChefServer.search(:client, query) do |raw|
             next unless raw.present?
             client = register(raw)
             Chef::Log.debug("Loaded #{client}")

lib/ironfan/provider/chef/node.rb

@@ -88,8 +88,8 @@ def conterminous_with_machine?
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ironfan.substep(cluster ? cluster.name : 'all', "nodes")
-          ChefServer.search(:node,"name:#{cluster.name}-*") do |raw|
+          query = cluster && "name:#{cluster.name}-*"
+          ChefServer.search(:node, query) do |raw|
             next unless raw.present?
             node = register(raw)
             Chef::Log.debug("Loaded #{node}")

lib/ironfan/provider/chef/role.rb

@@ -46,9 +46,9 @@ def to_s
         #
         # Discovery
         #
-        def self.load!(cluster)
-          Ironfan.substep(cluster ? cluster.name : 'all', "roles")
-          ChefServer.search(:role,"name:#{cluster.name}_*") do |raw|
+        def self.load!(cluster=nil)
+          query = cluster && "name:#{cluster.name}-*"
+          ChefServer.search(:role,query) do |raw|
             next unless raw.present?
             role = register(raw)
             Chef::Log.debug("Loaded #{role}")

lib/ironfan/provider/ec2/ebs_volume.rb

@@ -49,9 +49,9 @@ def ephemeral_device?
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ironfan.substep(cluster ? cluster.name : 'all', "volumes")
           Ec2.connection.volumes.each do |vol|
             next if vol.blank?
+            next if %w[deleting deleted error].include?(vol.state.to_s)
             ebs = EbsVolume.new(:adaptee => vol)
             # Already have a volume by this name
             if recall? ebs.name

lib/ironfan/provider/ec2/key_pair.rb

@@ -33,7 +33,6 @@ def to_s
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ironfan.substep(cluster ? cluster.name : 'all', "keypairs")
           Ec2.connection.key_pairs.each do |keypair|
             register keypair unless keypair.blank?
             Chef::Log.debug("Loaded <%-15s %s>" % [handle, keypair.name])

lib/ironfan/provider/ec2/machine.rb

@@ -102,18 +102,18 @@ def to_s
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ironfan.substep(cluster ? cluster.name : 'all', "machines")
           Ec2.connection.servers.each do |fs|
             machine = new(:adaptee => fs)
-            if recall? machine.name
+            if (not machine.created?)
+              next unless Ironfan.chef_config[:include_terminated]
+              remember machine, :append_id => "terminated:#{machine.id}"
+            elsif recall? machine.name
               raise 'duplicate'
               machine.bogus <<                 :duplicate_machines
               recall(machine.name).bogus <<    :duplicate_machines
               remember machine, :append_id => "duplicate:#{machine.id}"
-            elsif machine.created?
+            else # never seen it
               remember machine
-            else
-              remember machine, :append_id => "terminated:#{machine.id}"
             end
             Chef::Log.debug("Loaded #{machine}")
           end
@@ -168,7 +168,7 @@ def self.create!(computer)
             'name' =>         computer.name,
             'Name' =>         computer.name,
           }
-          Ec2.ensure_tags(tags,computer.machine)
+          Ec2.ensure_tags(tags, computer.machine)
 
           # register the new volumes for later save!, and tag appropriately
           computer.machine.volumes.each do |v|

lib/ironfan/provider/ec2/placement_group.rb

@@ -15,8 +15,7 @@ def to_s
           "<%-15s %-12s %-12s>" % [ self.class.handle, '', name ]
         end
 
-        def self.load!(cluster)
-          Ironfan.substep(cluster ? cluster.name : 'all', "placement groups")
+        def self.load!(cluster=nil)
           result = Ec2.connection.describe_placement_groups
           result.body["placementGroupSet"].each do |group|
             register group unless group.blank?

lib/ironfan/provider/ec2/security_group.rb

@@ -24,8 +24,6 @@ def self.expected_ids(computer)
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ironfan.substep(cluster ? cluster.name : 'all', "security groups")
-
           Ec2.connection.security_groups.each do |raw|
             next if raw.blank?
             sg = SecurityGroup.new(:adaptee => raw)