Fixes #16542 - More thorough pulp services check (Katello#6314)

pulp should be checked to be fully functional by checking

```
verify messaging_connection is True
verify database_connection is True
verify at least one scheduler@ entry is present
verify at least one resource_manager@ entry is present
verify at least one reserved_resource_worker-*@ is present
```

app/models/katello/ping.rb

@@ -117,10 +117,6 @@ def pulp_without_auth(url)
         fail _("Pulp does not appear to be running.") if body.empty?
         json = JSON.parse(body)
 
-        if json['known_workers'].empty?
-          fail _("No pulp workers running.")
-        end
-
         if json['database_connection'] && json['database_connection']['connected'] != true
           fail _("Pulp database connection issue.")
         end
@@ -129,8 +125,21 @@ def pulp_without_auth(url)
           fail _("Pulp message bus connection issue.")
         end
 
+        unless all_pulp_workers_present?(json)
+          fail _("Not all necessary pulp workers running.")
+        end
+
         json
       end
+
+      def all_pulp_workers_present?(json)
+        worker_ids = json["known_workers"].collect { |worker| worker["_id"] }
+        return false unless worker_ids.any?
+        scheduler = worker_ids.any? { |worker| worker.include?("scheduler@") }
+        resource_manager = worker_ids.any? { |worker| worker.include?("resource_manager@") }
+        reservered_resource_worker = worker_ids.any? { |worker| worker =~ /reserved_resource_worker-./ }
+        scheduler && resource_manager && reservered_resource_worker
+      end
     end
   end
 end

test/models/ping_test.rb

@@ -0,0 +1,44 @@
+require 'katello_test_helper'
+require 'rest-client'
+
+module Katello
+  class PingTest < ActiveSupport::TestCase
+    def setup
+      @ok_pulp_status = {"known_workers" =>
+                          [
+                            {"_ns" => "workers", "last_heartbeat" => "2016-09-13T20:07:54Z", "_id" => "scheduler@chicken.example.com"},
+                            {"_ns" => "workers", "last_heartbeat" => "2016-09-13T20:08:21Z", "_id" => "resource_manager@chicken.example.com"},
+                            {"_ns" => "workers", "last_heartbeat" => "2016-09-13T20:08:20Z", "_id" => "reserved_resource_worker-1@chicken.example.com"},
+                            {"_ns" => "workers", "last_heartbeat" => "2016-09-13T20:08:20Z", "_id" => "reserved_resource_worker-0@chicken.example.com"}
+                          ],
+                         "messaging_connection" => {"connected" => true},
+                         "database_connection" => {"connected" => true},
+                         "api_version" => "2",
+                         "versions" => {"platform_version" => "2.9.1"
+                          }
+                        }
+    end
+
+    def test_all_workers_present_ok_status
+      assert Katello::Ping.all_pulp_workers_present?(@ok_pulp_status)
+    end
+
+    def test_all_workers_present_no_scheduler
+      no_scheduler_status = @ok_pulp_status
+      no_scheduler_status["known_workers"].delete_if { |x| x["_id"] == "scheduler@chicken.example.com" }
+      refute Katello::Ping.all_pulp_workers_present?(no_scheduler_status)
+    end
+
+    def test_all_workers_present_no_resource_manager
+      no_resource_manager_status = @ok_pulp_status
+      no_resource_manager_status["known_workers"].delete_if { |x| x["_id"] == "resource_manager@chicken.example.com" }
+      refute Katello::Ping.all_pulp_workers_present?(no_resource_manager_status)
+    end
+
+    def test_all_workers_present_no_reserved_resource_worker
+      no_reserved_resource_worker_status = @ok_pulp_status
+      no_reserved_resource_worker_status["known_workers"].delete_if { |x| x["_id"] =~ /reserved_resource_worker-./ }
+      refute Katello::Ping.all_pulp_workers_present?(no_reserved_resource_worker_status)
+    end
+  end
+end