oo-admin-repair should properly handle HA apps with deleted head gears

Bug 1218049
Bugzilla Link https://bugzilla.redhat.com/show_bug.cgi?id=1218049
oo-admin-repair previously attempted to resolve HA applications with missing head gears as though they were simply missing a web-proxy gear. When a head gear is missing, the application cannot be recovered. oo-admin-repair should offer to cleanly remove the application rather than attempt to fix it (and break it more in the process).

broker-util/oo-admin-repair

@@ -164,7 +164,7 @@ end
 
 def analyze_app(app, unresponsive_servers)
   framework = { :available => false, :scaled =>false, :unresponsive_gears => [],
-                :responsive_gears => [], :backup_available => false }
+                :responsive_gears => [], :backup_available => false, :ha_lost_head => false }
   db = { :available => false, :unresponsive_gears => [], :responsive_gears => [],
          :backup_available => false, :remove_features => [] }
   app_recoverable = false
@@ -202,6 +202,15 @@ def analyze_app(app, unresponsive_servers)
     end
   end if app and app.gears.present?
 
+  # If the head gear of an HA application is missing, application cannot be recovered
+  if app.ha && framework[:available] == true
+    head_gear = app.gears.select{|g| g.app_dns}[0]
+    if unresponsive_servers.include?(head_gear.server_identity)
+      framework[:available] = false
+      framework[:ha_lost_head] = true
+    end
+  end
+
   if !framework[:available] and !db[:available]
     framework[:backup_available] = true if framework[:scaled]
   elsif !framework[:available] and db[:available]
@@ -716,12 +725,29 @@ def repair_removed_nodes(available_servers, unresponsive_servers, auto_confirm)
     end
 
     apps_not_recoverable = []
+    ha_apps_without_head = []
     app_reports.each do |app_id, app_info|
       app, recoverable, framework, db = app_info[0], app_info[1], app_info[2], app_info[3]
-      if !recoverable and !framework[:backup_available] and !db[:backup_available]
+      if !recoverable and framework[:ha_lost_head]
+        ha_apps_without_head << app_id
+      elsif !recoverable and !framework[:backup_available] and !db[:backup_available]
         apps_not_recoverable << app_id
       end 
     end
+
+    unless ha_apps_without_head.blank?
+      puts "Found #{ha_apps_without_head.size} HA applications that cannot be recovered due to missing head gear."
+      ha_apps_without_head.each { |app_id| puts "#{app_reports[app_id][0].name} (id: #{app_id})" }
+      puts ""
+      puts "These apps should be re-created and any existing aliases moved for recovery before deletion."
+      print "Do you want to delete all of them now [yes/no]: "
+      if auto_confirm or (auto_confirm.nil? and gets.chomp.to_b)
+        delete_apps(app_reports, ha_apps_without_head)
+      end
+      app_reports.delete_if { |k,v| ha_apps_without_head.include?(k) }
+      puts ""
+    end
+
     unless apps_not_recoverable.blank?
       puts "Found #{apps_not_recoverable.size} unresponsive scalable apps that can not be recovered."
       apps_not_recoverable.each { |app_id| puts "#{app_reports[app_id][0].name} (id: #{app_id})" }

broker/test/functional_ext/removed_nodes_app_fixup_test.rb

@@ -153,32 +153,14 @@ def test_unresponsive_apps
     assert_not_nil usage.end_time
 
     #test_scalable_app_ha_framework_gear_down
-    assert_equal(1, Application.where(canonical_name: @appnames[4].downcase).count)
-    app = Application.find_by(canonical_name: @appnames[4].downcase)
-    assert_equal(1, app.group_instances[0].gears.size)
-    assert_equal(3, UsageRecord.where(user_id: @cu._id, app_name: @appnames[4]).count)
+    assert_equal(0, Application.where(canonical_name: @appnames[4].downcase).count)
+    assert_equal(4, UsageRecord.where(user_id: @cu._id, app_name: @appnames[4]).count)
     assert_equal(2, Usage.where(user_id: @cu._id, app_name: @appnames[4]).count)
-    assert_equal(2, UsageRecord.where(gear_id: gear4).count)
-    assert_equal(1, Usage.where(gear_id: gear4).count)
-    usage = Usage.find_by(gear_id: gear4)
-    assert_not_nil usage.begin_time
-    assert_not_nil usage.end_time
 
     #test_scalable_app_ha_framework_gear_down_db_down
-    assert_equal(1, Application.where(canonical_name: @appnames[5].downcase).count)
-    app = Application.find_by(canonical_name: @appnames[5].downcase)
-    assert_equal(1, app.group_instances.size)
-    assert_equal(1, app.group_instances[0].gears.size)
-    assert_equal(5, UsageRecord.where(user_id: @cu._id, app_name: @appnames[5]).count)
-    assert_equal(3, Usage.where(user_id: @cu._id, app_name: @appnames[5]).count)
-    [gear5_1, gear5_2].each do |gear_id|
-      assert_equal(2, UsageRecord.where(gear_id: gear_id).count)
-      assert_equal(2, UsageRecord.where(gear_id: gear_id).count)
-      assert_equal(1, Usage.where(gear_id: gear_id).count)
-      usage = Usage.find_by(gear_id: gear_id)
-      assert_not_nil usage.begin_time
-      assert_not_nil usage.end_time
-    end
+    assert_equal(0, Application.where(canonical_name: @appnames[5].downcase).count)
+    assert_equal(4, UsageRecord.where(user_id: @cu._id, app_name: @appnames[5]).count)
+    assert_equal(2, Usage.where(user_id: @cu._id, app_name: @appnames[5]).count)
 
     #test_scalable_app_no_ha_scaled_up_head_gear_down
     assert_equal(0, Application.where(canonical_name: @appnames[6].downcase).count)