better reporting and processing of healing resuls, based on text tabl…

…es. numerous small improvements.

bin/heal

@@ -4,8 +4,12 @@ require 'rubygems'
 require 'trollop'
 require File.dirname(__FILE__)+'/../lib/healing'
 
+commands = 'ideal scan terminate heal volumes upload diagnose prune'.split
 
-commands = ['ideal','scan','terminate','heal']
+matches = commands.select { |c| c.match /^#{ARGV[0]}/ }
+if matches.size==1
+  ARGV[0] = matches.first
+end
 
 parser = nil
 opts = Trollop::options do
@@ -16,6 +20,13 @@ opts = Trollop::options do
   banner "\nHealing is a cloud configuration system."
 end
 
+
+if matches.size>1
+  puts "Which command? #{matches.join(', ')}"
+  exit
+end
+
+
 cmd = ARGV[0]
 
 unless cmd && commands.include?(cmd)
@@ -25,19 +36,7 @@ unless cmd && commands.include?(cmd)
   exit
 end
 
+#run subcommand:
 cmd = ARGV.shift
 path = Healing::BASE+'/bin/heal-'+cmd
-load path   #run subcommand
-
-
-
-#require 'healing'
-
-#h = Healing::App::Admin.new
-#h.describe
-
-#require 'lib/healing'
-#require 'lib/app/admin'
-
-#h = Healing::App::Admin.new
-#h.heal
+load path

bin/heal-diagnose

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+
+parser = nil
+opts = Trollop::options do
+  opt :ideal, "ideal file", :type => :string, :default => 'ideal.rb'
+  parser = self
+end
+
+h = Healing::App::Admin.new
+h.load_ideal opts[:ideal]
+h.diagnose

bin/heal-diagnose-local

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__)+'/../lib/healing'
+
+#FIXME this is a workaround, since /etc/profile is not loaded when we go through Net::SSH, for some reason,
+#not are any of the other bash config files.....
+ENV['PATH'] = "#{ENV['PATH']}:/var/lib/gems/1.8/bin"
+
+
+h = Healing::App::Worker.new
+h.load_ideal 'ideal.rb'
+h.diagnose

bin/heal-heal

@@ -2,7 +2,6 @@
 
 require 'rubygems'
 require 'trollop'
-#require File.dirname(__FILE__)+'/../lib/healing'
 
 parser = nil
 opts = Trollop::options do

bin/heal-ideal

@@ -2,7 +2,6 @@
 
 require 'rubygems'
 require 'trollop'
-#require File.dirname(__FILE__)+'/../lib/healing'
 
 parser = nil
 opts = Trollop::options do

bin/heal-prune

@@ -1,7 +1,14 @@
 #!/usr/bin/env ruby
 
-require 'lib/healing'
-require 'lib/app/admin'
+require 'rubygems'
+require 'trollop'
+
+parser = nil
+opts = Trollop::options do
+  opt :ideal, "ideal file", :type => :string, :default => 'ideal.rb'
+  parser = self
+end
 
 h = Healing::App::Admin.new
-h.prune
+h.load_ideal opts[:ideal]
+h.prune

bin/heal-scan

@@ -2,7 +2,6 @@
 
 require 'rubygems'
 require 'trollop'
-#require File.dirname(__FILE__)+'/../lib/healing'
 
 parser = nil
 opts = Trollop::options do

bin/heal-terminate

@@ -2,7 +2,6 @@
 
 require 'rubygems'
 require 'trollop'
-#require File.dirname(__FILE__)+'/../lib/healing'
 
 parser = nil
 opts = Trollop::options do

bin/heal-upload

@@ -1,7 +1,14 @@
 #!/usr/bin/env ruby
 
-require 'lib/healing'
-require 'lib/app/admin'
+require 'rubygems'
+require 'trollop'
+
+parser = nil
+opts = Trollop::options do
+  opt :ideal, "ideal file", :type => :string, :default => 'ideal.rb'
+  parser = self
+end
 
 h = Healing::App::Admin.new
-h.upload
+h.load_ideal opts[:ideal]
+h.upload

bin/heal-volumes

@@ -1,7 +1,14 @@
 #!/usr/bin/env ruby
 
-require 'lib/healing'
-require 'lib/app/admin'
+require 'rubygems'
+require 'trollop'
+
+parser = nil
+opts = Trollop::options do
+  opt :ideal, "ideal file", :type => :string, :default => 'ideal.rb'
+  parser = self
+end
 
 h = Healing::App::Admin.new
+h.load_ideal opts[:ideal]
 h.volumes

docs/notes

@@ -218,3 +218,69 @@ ssh						now we're at the instance
 bash					$_ = ]     $_ = hBc
 ruby
 backticks				here both $_ and $_ are empty, and PATH is wrong
+
+
+gems often install other gems they depend on. if we skip a gem because it's already installed, we might inadvertedly prevent other needed gems from being installed......
+
+
+ideas:
+split log files into raw command line text, and an tree-overview status overview.
+have each heal() method return a status, so we can collect all failed elements, and display them to the user, organized by instances
+after a healing, report number of healed, skipped, failed elements
+diagnose command, run healed? on all elements, and present a summary, or perhaps full trees
+
+
+
+
+we heal a cloud. we send the ideal to the bunch of instances, and start the healing process on each. the worker on an instance runs, and output is produced:
+
+- commands and their command line output
+- healing status lines
+
+all this stuff is going back to the admin app and saved in separate log files.
+however, it would be nice to be able to output a summary to the user. how would this be done? certainly it involves gathering info from all the instances.
+
+idea for showing a summary of a healing:
+
++----+----+------------+---------------------------------------+
+| Ok | Er | Cloud      | Element                               |
++----+----+------------+---------------------------------------+ 
+| 2  | 2  | dev        | Dir: /shared                          |
+| 1  |    | dev/db     | File: /db                             |
+|    | 1  | dev/slave  | File: /slave                          |
+| 2  |    | dev/app    | Dir: /xx                              |
+| 1  | 1  | dev/app    |  File: /xx/cool.txt                   |
++----+----+------------+---------------------------------------+
+
+only elements are shown, not the clouds themselves. cloud paths are shown in a separate column, and elements are only indented relative to their cloud.
+each row shows the number of instances that the element was healed on successfully and with error.
+
+if i can print a similarly formatted extract when healing on each instance, i should be able to collect them into a single summary.
+
+dev/db:
++----+----+------------+---------------------------------------+
+| Ok | Er | Cloud      | Element                               |
++----+----+------------+---------------------------------------+ 
+| 1  |    | dev        | Dir: /shared                          |
+| 1  |    | dev/db     | File: /db                             |
++----+----+------------+---------------------------------------+
+
+dev/slave:
++----+----+------------+---------------------------------------+
+| Ok | Er | Cloud      | Element                               |
++----+----+------------+---------------------------------------+ 
+| 1  |    | dev        | Dir: /shared                          |
+|    | 1  | dev/slave  | File: /slave                          |
++----+----+------------+---------------------------------------+
+
+dev/app:
++----+----+------------+---------------------------------------+
+| Ok | Er | Cloud      | Element                               |
++----+----+------------+---------------------------------------+ 
+|    | 2  | dev        | Dir: /shared                          |
+| 2  |    | dev/app    | Dir: /xx                              |
+| 1  | 1  | dev/app    |  File: /xx/cool.txt                   |
++----+----+------------+---------------------------------------+
+
+subclouds inhering parent cloud elements, so the instance healing summary includes these elements, but never elements from sibling clouds.
+

examples/drill.rb

@@ -8,6 +8,7 @@
 
   #core
   dir '/test'
+  dir '/test2', :mode => 0777     #remember to use octal (leading zero) for modes!
   file '/test/file', :content => 'once upon a time'
   line_in_file '/test/file', :content => 'fairies = 7'
   link '/test/file', '/test/fairies'

examples/rake.rb

@@ -0,0 +1,13 @@
+
+cloud :raking do
+  uuid '68u2hcjcdjj3'
+  key '/Users/emiltin/.ec2/testpair'
+  instances 1
+
+  mysql_ebs 'vol-4943a020'
+  rails_app '/poolparty_example', :repo => 'git://github.com/emiltin/poolparty_example.git', :environment => :production
+#  rubygem 'rake'
+  rake "stats", :base => '/poolparty_example'#, :flags => "RAILS_ENV=production"
+  rake "log:clear", :base => '/poolparty_example'
+end
+

ideal.rb

@@ -1,13 +1,22 @@
 
-cloud :raking do
-  uuid '68u2hcjcdjj3'
+cloud :dev do
+  uuid 'hjujj3j'
   key '/Users/emiltin/.ec2/testpair'
-  instances 1
 
-#  mysql_ebs 'vol-4943a020'
-#  rails_app '/poolparty_example', :repo => 'git://github.com/emiltin/poolparty_example.git', :environment => :production
-#  rubygem 'rake'
-  rake "stats", :base => '/poolparty_example'
-  rake "log:clear", :base => '/poolparty_example'
+  dir '/xx'
+
+  cloud :app do
+    instances 10
+    uuid '3jfjfkj3kr'
+    dir '/xx/app'
+  end
+
+  cloud :db do
+    uuid 'dlfkjl3rddd3333'
+    instances 1
+    dir '/xx/db'
+  end
+
 end
 
+

lib/app/admin.rb

@@ -4,6 +4,7 @@ module App
     class Admin < Base
 
       def describe
+  #      Healing::Structure::Rubygem.scan_gems
         @cloud.describe :recurse => true
       end
 
@@ -22,6 +23,12 @@ def terminate
 
       def heal
         @cloud.heal_remote
+        @cloud.report_remote
+      end
+
+      def diagnose
+        @cloud.diagnose_remote
+        @cloud.report_remote
       end
 
       def prune

lib/app/base.rb

@@ -12,7 +12,7 @@ def self.run_locally cmd, options={}
           err = stderr.read
           unless options[:quiet]
             puts out unless out=='' 
-            puts "stderr: #{err}" unless err==''
+            puts "*** ERROR ***: #{err}" unless err==''
           end
           return out
         end

lib/app/provisioner.rb

@@ -5,7 +5,10 @@ class Provisioner
       def initialize root
         @root = root
       end
-
+
+      def diagnose
+      end
+
       def heal
         if arm
           launch

lib/app/worker.rb

@@ -5,6 +5,7 @@ class Worker < Base
 
       def initialize
         load_cloud_uuid
+        Healing::Structure::Rubygem.scan_gems
         super
       end
 
@@ -27,9 +28,17 @@ def load_cloud_uuid
       end
 
       def heal
+        puts '------------------------------------------------------------'
         puts "Healing '#{@cloud.options.name}' instance..."
         @cloud.heal_from_root
-        puts "Done."
+        @cloud.root.report
+      end
+
+      def diagnose
+        puts '------------------------------------------------------------'
+        puts "Diagnosing '#{@cloud.options.name}' instance..."
+        @cloud.diagnose_from_root
+        @cloud.root.report
       end
     end
   end

lib/healing.rb

@@ -1,11 +1,18 @@
 module Healing
   BASE = File.dirname( File.dirname(__FILE__) )
 
-  Dir.glob(BASE+'/lib/utilities/*.rb') { |file| require file }     
+  #note: the order of the requires below is significant to avoid load errors
+
+  require 'fileutils'
+
+  require BASE+'/lib/utilities/options.rb'
+  require BASE+'/lib/utilities/table.rb'
+  require BASE+'/lib/utilities/reporter.rb'
+  require BASE+'/lib/utilities/threading.rb'
+
   Dir.glob(BASE+'/lib/core/*.rb') { |file| require file }     
 
 
-  #note: the order of the requires below is significant to avoid load errors
 
   require BASE+'/lib/structure/base.rb'
   require BASE+'/lib/structure/resources/resource'