From b3a41c426f82425c0c6e46accd6e8c5bb9c41186 Mon Sep 17 00:00:00 2001
From: Hideo Yamauchi <renayama19661014@ybb.ne.jp>
Date: Thu, 25 Sep 2014 10:09:41 +0900
Subject: [PATCH 01/29] Correction of the log output of the practice node of
 stonith.

Correction of a possible point of the influence.
---
 crmd/te_utils.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/crmd/te_utils.c b/crmd/te_utils.c
index 66ed1daa687..69c22e34877 100644
--- a/crmd/te_utils.c
+++ b/crmd/te_utils.c
@@ -126,19 +126,19 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event)
 
     if (st_event->result == pcmk_ok && safe_str_eq("on", st_event->action)) {
         crm_notice("%s was successfully unfenced by %s (at the request of %s)",
-                   st_event->target, st_event->executioner, st_event->origin);
+                   st_event->target, st_event->executioner ? st_event->executioner : "<anyone>", st_event->origin);
                 /* TODO: Hook up st_event->device */
         return;
 
     } else if (safe_str_eq("on", st_event->action)) {
         crm_err("Unfencing of %s by %s failed: %s (%d)",
-                st_event->target, st_event->executioner,
+                st_event->target, st_event->executioner ? st_event->executioner : "<anyone>",
                 pcmk_strerror(st_event->result), st_event->result);
         return;
 
     } else if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) {
         crm_crit("We were allegedly just fenced by %s for %s!",
-                 st_event->executioner, st_event->origin); /* Dumps blackbox if enabled */
+                 st_event->executioner ? st_event->executioner : "<anyone>", st_event->origin); /* Dumps blackbox if enabled */
 
         qb_log_fini(); /* Try to get the above log message to disk - somehow */
 

From a36144cd5fd8a8953d88b0589927b67dffa59569 Mon Sep 17 00:00:00 2001
From: Hideo Yamauchi <renayama19661014@ybb.ne.jp>
Date: Thu, 25 Sep 2014 10:10:06 +0900
Subject: [PATCH 02/29] Correction of the log output of the practice node of
 stonith.(2)

Revise one next correction.
*
https://github.com/ClusterLabs/pacemaker/commit/a932361589964545f76376009d7ff7a61f03cab3
---
 fencing/remote.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/fencing/remote.c b/fencing/remote.c
index 3f4f5ca2fea..834c440970a 100644
--- a/fencing/remote.c
+++ b/fencing/remote.c
@@ -280,8 +280,6 @@ remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup)
         xmlNode *ndata = get_xpath_object("//@" F_STONITH_DELEGATE, data, LOG_WARNING);
         if(ndata) {
             op->delegate = crm_element_value_copy(ndata, F_STONITH_DELEGATE);
-        } else {
-            op->delegate = crm_element_value_copy(data, F_ORIG);
         }
     }
 

From 2e2be87a7ef69c1fd8e7c53a6569e5707b54231d Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Wed, 1 Oct 2014 12:07:21 -0400
Subject: [PATCH 03/29] High: pengine: resource discovery mode for location
 constraints

---
 include/crm/msg_xml.h        |  2 ++
 include/crm/pengine/status.h |  2 ++
 pengine/allocate.c           |  2 +-
 pengine/constraints.c        |  6 ++++--
 pengine/native.c             | 27 +++++++++++++++++++++++++--
 pengine/pengine.h            |  7 +++++++
 pengine/utils.c              | 15 ++++++++++++++-
 pengine/utils.h              |  3 ++-
 xml/constraints-next.rng     | 13 +++++++++++++
 9 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
index 1ff425ca38c..8e5708cf85d 100644
--- a/include/crm/msg_xml.h
+++ b/include/crm/msg_xml.h
@@ -298,6 +298,8 @@
 #  define XML_CONS_TAG_RSC_SET		"resource_set"
 #  define XML_CONS_ATTR_SYMMETRICAL	"symmetrical"
 
+#  define XML_LOCATION_ATTR_DISCOVERY	"resource-discovery"
+
 #  define XML_COLOC_ATTR_SOURCE		"rsc"
 #  define XML_COLOC_ATTR_SOURCE_ROLE	"rsc-role"
 #  define XML_COLOC_ATTR_TARGET		"with-rsc"
diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
index d0ba85672f7..3708d7ef164 100644
--- a/include/crm/pengine/status.h
+++ b/include/crm/pengine/status.h
@@ -156,6 +156,7 @@ struct node_shared_s {
 struct node_s {
     int weight;
     gboolean fixed;
+    int rsc_discover_mode;
     int count;
     struct node_shared_s *details;
 };
@@ -252,6 +253,7 @@ struct resource_s {
     int migration_threshold;
 
     gboolean is_remote_node;
+    gboolean exclusive_discover;
 
     unsigned long long flags;
 
diff --git a/pengine/allocate.c b/pengine/allocate.c
index 8d02d9b6c2a..57d81ef451f 100644
--- a/pengine/allocate.c
+++ b/pengine/allocate.c
@@ -755,7 +755,7 @@ apply_system_health(pe_working_set_t * data_set)
             for (; gIter2 != NULL; gIter2 = gIter2->next) {
                 resource_t *rsc = (resource_t *) gIter2->data;
 
-                rsc2node_new(health_strategy, rsc, system_health, node, data_set);
+                rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
             }
         }
 
diff --git a/pengine/constraints.c b/pengine/constraints.c
index 1aa848ef087..88e382b83db 100644
--- a/pengine/constraints.c
+++ b/pengine/constraints.c
@@ -658,6 +658,7 @@ unpack_rsc_location(xmlNode * xml_obj, resource_t * rsc_lh, const char * role,
     const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *node = crm_element_value(xml_obj, XML_CIB_TAG_NODE);
+    const char *discovery = crm_element_value(xml_obj, XML_LOCATION_ATTR_DISCOVERY);
 
     if (rsc_lh == NULL) {
         /* only a warn as BSC adds the constraint then the resource */
@@ -676,7 +677,7 @@ unpack_rsc_location(xmlNode * xml_obj, resource_t * rsc_lh, const char * role,
         if (!match) {
             return FALSE;
         }
-        location = rsc2node_new(id, rsc_lh, score_i, match, data_set);
+        location = rsc2node_new(id, rsc_lh, score_i, discovery, match, data_set);
 
     } else {
         xmlNode *rule_xml = NULL;
@@ -720,6 +721,7 @@ unpack_rsc_location(xmlNode * xml_obj, resource_t * rsc_lh, const char * role,
             }
         }
     }
+
     return TRUE;
 }
 
@@ -958,7 +960,7 @@ generate_location_rule(resource_t * rsc, xmlNode * rule_xml, pe_working_set_t *
         do_and = FALSE;
     }
 
-    location_rule = rsc2node_new(rule_id, rsc, 0, NULL, data_set);
+    location_rule = rsc2node_new(rule_id, rsc, 0, NULL, NULL, data_set);
 
     if (location_rule == NULL) {
         return NULL;
diff --git a/pengine/native.c b/pengine/native.c
index 5db8c603c0f..00203445507 100644
--- a/pengine/native.c
+++ b/pengine/native.c
@@ -2097,9 +2097,14 @@ native_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
             other_node->weight = merge_weights(other_node->weight, node->weight);
 
         } else {
-            node_t *new_node = node_copy(node);
+            other_node = node_copy(node);
 
-            g_hash_table_insert(rsc->allowed_nodes, (gpointer) new_node->details->id, new_node);
+            g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node);
+        }
+
+        if (other_node->rsc_discover_mode < constraint->discover_mode) {
+            /* exclusive > never > always... always is default */
+            other_node->rsc_discover_mode = constraint->discover_mode;
         }
     }
 
@@ -2704,6 +2709,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
     char *key = NULL;
     action_t *probe = NULL;
     node_t *running = NULL;
+    node_t *allowed = NULL;
     resource_t *top = uber_parent(rsc);
 
     static const char *rc_master = NULL;
@@ -2780,6 +2786,23 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
         return FALSE;
     }
 
+    allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
+    if (rsc->exclusive_discover || top->exclusive_discover) {
+        if (allowed == NULL) {
+            /* exclusive discover is enabled and this node is not in the allowed list. */    
+            return FALSE;
+        } else if (allowed->rsc_discover_mode != discover_exclusive) {
+            /* exclusive discover is enabled and this node is not marked
+             * as a node this resource should be discovered on */ 
+            return FALSE;
+        }
+    }
+    if (allowed && allowed->rsc_discover_mode == discover_never) {
+        /* this resource is marked as not needing to be discovered on this node */
+        return FALSE;
+    }
+
+
     key = generate_op_key(rsc->id, RSC_STATUS, 0);
     probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
     update_action_flags(probe, pe_action_optional | pe_action_clear);
diff --git a/pengine/pengine.h b/pengine/pengine.h
index 653fadf4061..87fa1507c49 100644
--- a/pengine/pengine.h
+++ b/pengine/pengine.h
@@ -75,11 +75,18 @@ struct rsc_ticket_s {
     int role_lh;
 };
 
+enum rsc_discover_e {
+    discover_always = 0,
+    discover_never,
+    discover_exclusive,
+};
+
 struct rsc_to_node_s {
     const char *id;
     resource_t *rsc_lh;
 
     enum rsc_role_e role_filter;
+    enum rsc_discover_e discover_mode;
     GListPtr node_list_rh;      /* node_t* */
 };
 
diff --git a/pengine/utils.c b/pengine/utils.c
index 7dfaf95fe3f..9782cb61446 100644
--- a/pengine/utils.c
+++ b/pengine/utils.c
@@ -60,7 +60,8 @@ pe_free_rsc_to_node(GListPtr constraints)
 
 rsc_to_node_t *
 rsc2node_new(const char *id, resource_t * rsc,
-             int node_weight, node_t * foo_node, pe_working_set_t * data_set)
+             int node_weight, const char *discover_mode,
+             node_t * foo_node, pe_working_set_t * data_set)
 {
     rsc_to_node_t *new_con = NULL;
 
@@ -79,6 +80,18 @@ rsc2node_new(const char *id, resource_t * rsc,
         new_con->node_list_rh = NULL;
         new_con->role_filter = RSC_ROLE_UNKNOWN;
 
+
+        if (discover_mode == NULL || safe_str_eq(discover_mode, "always")) {
+            new_con->discover_mode = discover_always;
+        } else if (safe_str_eq(discover_mode, "never")) {
+            new_con->discover_mode = discover_never;
+        } else if (safe_str_eq(discover_mode, "exclusive")) {
+            new_con->discover_mode = discover_exclusive;
+            rsc->exclusive_discover = TRUE;
+        } else {
+            pe_err("Invalid %s value %s in location constraint", XML_LOCATION_ATTR_DISCOVERY, discover_mode);
+        }
+
         if (foo_node != NULL) {
             node_t *copy = node_copy(foo_node);
 
diff --git a/pengine/utils.h b/pengine/utils.h
index 1939d409561..5142e689981 100644
--- a/pengine/utils.h
+++ b/pengine/utils.h
@@ -23,7 +23,8 @@ extern rsc_colocation_t *invert_constraint(rsc_colocation_t * constraint);
 
 extern rsc_to_node_t *copy_constraint(rsc_to_node_t * constraint);
 
-extern rsc_to_node_t *rsc2node_new(const char *id, resource_t * rsc, int weight, node_t * node,
+extern rsc_to_node_t *rsc2node_new(const char *id, resource_t * rsc, int weight,
+                                   const char *discovery_mode, node_t * node,
                                    pe_working_set_t * data_set);
 
 extern void pe_free_rsc_to_node(GListPtr constraints);
diff --git a/xml/constraints-next.rng b/xml/constraints-next.rng
index 051603d93e9..90f5b0f40a4 100644
--- a/xml/constraints-next.rng
+++ b/xml/constraints-next.rng
@@ -49,6 +49,11 @@
       <optional>
 	<ref name="element-lifetime"/>
       </optional>
+      <optional>
+	<attribute name="resource-discovery">
+	  <ref name="attribute-discovery"/>
+	</attribute>
+      </optional>
     </element>
   </define>
 
@@ -206,6 +211,14 @@
     </element>
   </define>
 
+  <define name="attribute-discovery">
+    <choice>
+      <value>always</value>
+      <value>never</value>
+      <value>exclusive</value>
+    </choice>
+  </define>
+
   <define name="attribute-actions">
     <choice>
       <value>start</value>

From 6b0a26331f53a49251ee2aa31fda05d32ecf6b25 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Wed, 1 Oct 2014 14:13:27 -0400
Subject: [PATCH 04/29] Low: pengine: resource-discovery regression test

---
 pengine/regression.sh                     |    4 +
 pengine/test10/resource-discovery.dot     |  185 ++++
 pengine/test10/resource-discovery.exp     | 1041 +++++++++++++++++++++
 pengine/test10/resource-discovery.scores  |  199 ++++
 pengine/test10/resource-discovery.summary |  124 +++
 pengine/test10/resource-discovery.xml     |  185 ++++
 6 files changed, 1738 insertions(+)
 create mode 100644 pengine/test10/resource-discovery.dot
 create mode 100644 pengine/test10/resource-discovery.exp
 create mode 100644 pengine/test10/resource-discovery.scores
 create mode 100644 pengine/test10/resource-discovery.summary
 create mode 100644 pengine/test10/resource-discovery.xml

diff --git a/pengine/regression.sh b/pengine/regression.sh
index bdc7d3a90fe..8acf890fea9 100755
--- a/pengine/regression.sh
+++ b/pengine/regression.sh
@@ -768,5 +768,9 @@ do_test remote-disable         "Disable a baremetal remote-node"
 do_test remote-orphaned        "Properly shutdown orphaned connection resource"
 do_test remote-recover         "Recover connection resource after cluster-node fails."
 do_test remote-stale-node-entry "Make sure we properly handle leftover remote-node entries in the node section"
+
+echo ""
+do_test resource-discovery "Exercises resource-discovery location constraint option."
+
 echo ""
 test_results
diff --git a/pengine/test10/resource-discovery.dot b/pengine/test10/resource-discovery.dot
new file mode 100644
index 00000000000..efb2434047c
--- /dev/null
+++ b/pengine/test10/resource-discovery.dot
@@ -0,0 +1,185 @@
+ digraph "g" {
+"FAKE1_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
+"FAKE1_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
+"FAKE1_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
+"FAKE1_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE1_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE1_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE1_start_0 18node2" -> "FAKE1_monitor_60000 18node2" [ style = bold]
+"FAKE1_start_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE2_monitor_60000 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE2_start_0 18node3" -> "FAKE2_monitor_60000 18node3" [ style = bold]
+"FAKE2_start_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE2_stop_0 18node2" -> "FAKE2_start_0 18node3" [ style = bold]
+"FAKE2_stop_0 18node2" -> "all_stopped" [ style = bold]
+"FAKE2_stop_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE3_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE3_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE3_monitor_60000 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE3_start_0 18node3" -> "FAKE3_monitor_60000 18node3" [ style = bold]
+"FAKE3_start_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE3_stop_0 18builder" -> "FAKE3_start_0 18node3" [ style = bold]
+"FAKE3_stop_0 18builder" -> "all_stopped" [ style = bold]
+"FAKE3_stop_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE4_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE4_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE4_monitor_60000 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE4_start_0 18node4" -> "FAKE4_monitor_60000 18node4" [ style = bold]
+"FAKE4_start_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE4_stop_0 18node1" -> "FAKE4_start_0 18node4" [ style = bold]
+"FAKE4_stop_0 18node1" -> "all_stopped" [ style = bold]
+"FAKE4_stop_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
+"FAKE5_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
+"FAKE5_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
+"FAKE5_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE5_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE5_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
+"FAKE5_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKE5_start_0 remote1" -> "FAKE5_monitor_60000 remote1" [ style = bold]
+"FAKE5_start_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:0_start_0 18builder" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:1_start_0 18node1" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:2_start_0 18node2" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:3_start_0 18node4" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:4_start_0 remote1" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:5_start_0 18node3" [ style = bold]
+"FAKECLONE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE1:0_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:0_start_0 18builder" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:0_start_0 18builder" -> "FAKECLONE1:0_monitor_60000 18builder" [ style = bold]
+"FAKECLONE1:0_start_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:1_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:1_start_0 18node1" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:1_start_0 18node1" -> "FAKECLONE1:1_monitor_60000 18node1" [ style = bold]
+"FAKECLONE1:1_start_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:2_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:2_start_0 18node2" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:2_start_0 18node2" -> "FAKECLONE1:2_monitor_60000 18node2" [ style = bold]
+"FAKECLONE1:2_start_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:3_monitor_60000 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:3_start_0 18node4" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:3_start_0 18node4" -> "FAKECLONE1:3_monitor_60000 18node4" [ style = bold]
+"FAKECLONE1:3_start_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:4_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
+"FAKECLONE1:4_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:4_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:4_start_0 remote1" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:4_start_0 remote1" -> "FAKECLONE1:4_monitor_60000 remote1" [ style = bold]
+"FAKECLONE1:4_start_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:5_monitor_60000 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:5_start_0 18node3" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:5_start_0 18node3" -> "FAKECLONE1:5_monitor_60000 18node3" [ style = bold]
+"FAKECLONE1:5_start_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:0_start_0 18builder" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:1_start_0 18node1" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:2_start_0 18node2" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:3_start_0 18node4" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:4_start_0 remote1" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:5_start_0 18node3" [ style = bold]
+"FAKECLONE2-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2:0_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
+"FAKECLONE2:0_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:0_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:0_start_0 18builder" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:0_start_0 18builder" -> "FAKECLONE2:0_monitor_60000 18builder" [ style = bold]
+"FAKECLONE2:0_start_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:1_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
+"FAKECLONE2:1_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:1_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:1_start_0 18node1" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:1_start_0 18node1" -> "FAKECLONE2:1_monitor_60000 18node1" [ style = bold]
+"FAKECLONE2:1_start_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:2_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:2_start_0 18node2" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:2_start_0 18node2" -> "FAKECLONE2:2_monitor_60000 18node2" [ style = bold]
+"FAKECLONE2:2_start_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:3_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKECLONE2:3_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:3_monitor_60000 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:3_start_0 18node4" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:3_start_0 18node4" -> "FAKECLONE2:3_monitor_60000 18node4" [ style = bold]
+"FAKECLONE2:3_start_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:4_monitor_0 remote1" -> "probe_complete remote1" [ style = bold]
+"FAKECLONE2:4_monitor_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:4_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:4_start_0 remote1" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:4_start_0 remote1" -> "FAKECLONE2:4_monitor_60000 remote1" [ style = bold]
+"FAKECLONE2:4_start_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:5_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKECLONE2:5_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:5_monitor_60000 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:5_start_0 18node3" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:5_start_0 18node3" -> "FAKECLONE2:5_monitor_60000 18node3" [ style = bold]
+"FAKECLONE2:5_start_0 18node3" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"probe_complete 18builder" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18builder" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node1" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node1" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node2" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node2" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node3" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node3" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node4" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node4" [ style=bold color="green" fontcolor="black"]
+"probe_complete remote1" -> "probe_complete" [ style = bold]
+"probe_complete remote1" [ style=bold color="green" fontcolor="black"]
+"probe_complete" -> "FAKE1_start_0 18node2" [ style = bold]
+"probe_complete" -> "FAKE2_stop_0 18node2" [ style = bold]
+"probe_complete" -> "FAKE3_start_0 18node3" [ style = bold]
+"probe_complete" -> "FAKE3_stop_0 18builder" [ style = bold]
+"probe_complete" -> "FAKE4_start_0 18node4" [ style = bold]
+"probe_complete" -> "FAKE4_stop_0 18node1" [ style = bold]
+"probe_complete" -> "FAKE5_start_0 remote1" [ style = bold]
+"probe_complete" -> "FAKECLONE1:0_start_0 18builder" [ style = bold]
+"probe_complete" -> "FAKECLONE1:1_start_0 18node1" [ style = bold]
+"probe_complete" -> "FAKECLONE1:2_start_0 18node2" [ style = bold]
+"probe_complete" -> "FAKECLONE1:3_start_0 18node4" [ style = bold]
+"probe_complete" -> "FAKECLONE1:4_start_0 remote1" [ style = bold]
+"probe_complete" -> "FAKECLONE1:5_start_0 18node3" [ style = bold]
+"probe_complete" -> "FAKECLONE2:0_start_0 18builder" [ style = bold]
+"probe_complete" -> "FAKECLONE2:1_start_0 18node1" [ style = bold]
+"probe_complete" -> "FAKECLONE2:2_start_0 18node2" [ style = bold]
+"probe_complete" -> "FAKECLONE2:3_start_0 18node4" [ style = bold]
+"probe_complete" -> "FAKECLONE2:4_start_0 remote1" [ style = bold]
+"probe_complete" -> "FAKECLONE2:5_start_0 18node3" [ style = bold]
+"probe_complete" [ style=bold color="green" fontcolor="orange"]
+"probe_nodes_complete" -> "remote1_start_0 18builder" [ style = bold]
+"probe_nodes_complete" [ style=bold color="green" fontcolor="orange"]
+"remote1_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"remote1_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"remote1_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"remote1_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"remote1_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
+"remote1_start_0 18builder" -> "FAKE5_monitor_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKE5_monitor_60000 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKE5_start_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE1:4_monitor_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE1:4_monitor_60000 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE1:4_start_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE2:4_monitor_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE2:4_monitor_60000 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE2:4_start_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "remote1_monitor_60000 18builder" [ style = bold]
+"remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"]
+"shooter_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"shooter_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"shooter_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"shooter_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/resource-discovery.exp b/pengine/test10/resource-discovery.exp
new file mode 100644
index 00000000000..2770f4ec95d
--- /dev/null
+++ b/pengine/test10/resource-discovery.exp
@@ -0,0 +1,1041 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="27" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="shooter" class="stonith" type="fence_xvm"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="20" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="shooter" class="stonith" type="fence_xvm"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="2">
+    <action_set>
+      <rsc_op id="40" operation="monitor" operation_key="remote1_monitor_60000" on_node="18builder" on_node_uuid="5">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="3">
+    <action_set>
+      <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="7" operation="probe_nodes_complete" operation_key="probe_nodes_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="4">
+    <action_set>
+      <rsc_op id="28" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="5">
+    <action_set>
+      <rsc_op id="21" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="6">
+    <action_set>
+      <rsc_op id="42" operation="monitor" operation_key="FAKE1_monitor_60000" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="41" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="7">
+    <action_set>
+      <rsc_op id="41" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="8">
+    <action_set>
+      <rsc_op id="29" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="9">
+    <action_set>
+      <rsc_op id="22" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="10">
+    <action_set>
+      <rsc_op id="17" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="11">
+    <action_set>
+      <rsc_op id="13" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="12">
+    <action_set>
+      <rsc_op id="9" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="13">
+    <action_set>
+      <rsc_op id="45" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="44" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="14">
+    <action_set>
+      <rsc_op id="44" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="43" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="15">
+    <action_set>
+      <rsc_op id="43" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="16">
+    <action_set>
+      <rsc_op id="48" operation="monitor" operation_key="FAKE3_monitor_60000" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="47" operation="start" operation_key="FAKE3_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="17">
+    <action_set>
+      <rsc_op id="47" operation="start" operation_key="FAKE3_start_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="46" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="18">
+    <action_set>
+      <rsc_op id="46" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="19">
+    <action_set>
+      <rsc_op id="23" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="20">
+    <action_set>
+      <rsc_op id="51" operation="monitor" operation_key="FAKE4_monitor_60000" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="50" operation="start" operation_key="FAKE4_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="21">
+    <action_set>
+      <rsc_op id="50" operation="start" operation_key="FAKE4_start_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="49" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="22">
+    <action_set>
+      <rsc_op id="49" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="23">
+    <action_set>
+      <rsc_op id="30" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="24">
+    <action_set>
+      <rsc_op id="53" operation="monitor" operation_key="FAKE5_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="52" operation="start" operation_key="FAKE5_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="25">
+    <action_set>
+      <rsc_op id="52" operation="start" operation_key="FAKE5_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="26">
+    <action_set>
+      <rsc_op id="34" operation="monitor" operation_key="FAKE5_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="27">
+    <action_set>
+      <rsc_op id="31" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="28">
+    <action_set>
+      <rsc_op id="24" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="29">
+    <action_set>
+      <rsc_op id="18" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="30">
+    <action_set>
+      <rsc_op id="14" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="31">
+    <action_set>
+      <rsc_op id="10" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="32">
+    <action_set>
+      <rsc_op id="55" operation="monitor" operation_key="FAKECLONE1:0_monitor_60000" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="54" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="33">
+    <action_set>
+      <rsc_op id="54" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="34">
+    <action_set>
+      <rsc_op id="57" operation="monitor" operation_key="FAKECLONE1:1_monitor_60000" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="56" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="35">
+    <action_set>
+      <rsc_op id="56" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="36">
+    <action_set>
+      <rsc_op id="59" operation="monitor" operation_key="FAKECLONE1:2_monitor_60000" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="58" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="37">
+    <action_set>
+      <rsc_op id="58" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="38">
+    <action_set>
+      <rsc_op id="61" operation="monitor" operation_key="FAKECLONE1:3_monitor_60000" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="39">
+    <action_set>
+      <rsc_op id="60" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="40">
+    <action_set>
+      <rsc_op id="63" operation="monitor" operation_key="FAKECLONE1:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="62" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="41">
+    <action_set>
+      <rsc_op id="62" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="42">
+    <action_set>
+      <rsc_op id="35" operation="monitor" operation_key="FAKECLONE1:4_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="43">
+    <action_set>
+      <rsc_op id="65" operation="monitor" operation_key="FAKECLONE1:5_monitor_60000" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="64" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="44">
+    <action_set>
+      <rsc_op id="64" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="45" priority="1000000">
+    <action_set>
+      <pseudo_event id="67" operation="running" operation_key="FAKECLONE1-clone_running_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="54" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="56" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="58" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="62" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="64" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="46">
+    <action_set>
+      <pseudo_event id="66" operation="start" operation_key="FAKECLONE1-clone_start_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="47">
+    <action_set>
+      <rsc_op id="71" operation="monitor" operation_key="FAKECLONE2:0_monitor_60000" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="70" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="48">
+    <action_set>
+      <rsc_op id="70" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="49">
+    <action_set>
+      <rsc_op id="11" operation="monitor" operation_key="FAKECLONE2:0_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="50">
+    <action_set>
+      <rsc_op id="73" operation="monitor" operation_key="FAKECLONE2:1_monitor_60000" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="72" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="51">
+    <action_set>
+      <rsc_op id="72" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="52">
+    <action_set>
+      <rsc_op id="15" operation="monitor" operation_key="FAKECLONE2:1_monitor_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="53">
+    <action_set>
+      <rsc_op id="75" operation="monitor" operation_key="FAKECLONE2:2_monitor_60000" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="74" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="54">
+    <action_set>
+      <rsc_op id="74" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="55">
+    <action_set>
+      <rsc_op id="77" operation="monitor" operation_key="FAKECLONE2:3_monitor_60000" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="56">
+    <action_set>
+      <rsc_op id="76" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="57">
+    <action_set>
+      <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="58">
+    <action_set>
+      <rsc_op id="79" operation="monitor" operation_key="FAKECLONE2:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="78" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="59">
+    <action_set>
+      <rsc_op id="78" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="60">
+    <action_set>
+      <rsc_op id="36" operation="monitor" operation_key="FAKECLONE2:4_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="61">
+    <action_set>
+      <rsc_op id="81" operation="monitor" operation_key="FAKECLONE2:5_monitor_60000" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="80" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="62">
+    <action_set>
+      <rsc_op id="80" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="63">
+    <action_set>
+      <rsc_op id="25" operation="monitor" operation_key="FAKECLONE2:5_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="64" priority="1000000">
+    <action_set>
+      <pseudo_event id="83" operation="running" operation_key="FAKECLONE2-clone_running_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="70" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="72" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="74" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="78" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="80" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="65">
+    <action_set>
+      <pseudo_event id="82" operation="start" operation_key="FAKECLONE2-clone_start_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="66" priority="1000000">
+    <action_set>
+      <rsc_op id="33" operation="probe_complete" operation_key="probe_complete-remote1" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="34" operation="monitor" operation_key="FAKE5_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="35" operation="monitor" operation_key="FAKECLONE1:4_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="36" operation="monitor" operation_key="FAKECLONE2:4_monitor_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="67" priority="1000000">
+    <action_set>
+      <rsc_op id="26" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="27" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="28" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="29" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="30" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="31" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="68" priority="1000000">
+    <action_set>
+      <rsc_op id="19" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="20" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="21" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="22" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="23" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="24" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="25" operation="monitor" operation_key="FAKECLONE2:5_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="69" priority="1000000">
+    <action_set>
+      <rsc_op id="16" operation="probe_complete" operation_key="probe_complete-18node2" on_node="18node2" on_node_uuid="2">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="17" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="18" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="70" priority="1000000">
+    <action_set>
+      <rsc_op id="12" operation="probe_complete" operation_key="probe_complete-18node1" on_node="18node1" on_node_uuid="1">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="13" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="14" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="15" operation="monitor" operation_key="FAKECLONE2:1_monitor_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="71" priority="1000000">
+    <action_set>
+      <rsc_op id="8" operation="probe_complete" operation_key="probe_complete-18builder" on_node="18builder" on_node_uuid="5">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="9" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="10" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="11" operation="monitor" operation_key="FAKECLONE2:0_monitor_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="72">
+    <action_set>
+      <pseudo_event id="7" operation="probe_nodes_complete" operation_key="probe_nodes_complete">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="8" operation="probe_complete" operation_key="probe_complete-18builder" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="12" operation="probe_complete" operation_key="probe_complete-18node1" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="16" operation="probe_complete" operation_key="probe_complete-18node2" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="19" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="26" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="73">
+    <action_set>
+      <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="33" operation="probe_complete" operation_key="probe_complete-remote1" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="74">
+    <action_set>
+      <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="43" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="46" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="49" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+</transition_graph>
diff --git a/pengine/test10/resource-discovery.scores b/pengine/test10/resource-discovery.scores
new file mode 100644
index 00000000000..e1fa78eaa13
--- /dev/null
+++ b/pengine/test10/resource-discovery.scores
@@ -0,0 +1,199 @@
+Allocation scores:
+clone_color: FAKECLONE1-clone allocation score on 18builder: 0
+clone_color: FAKECLONE1-clone allocation score on 18node1: 0
+clone_color: FAKECLONE1-clone allocation score on 18node2: 0
+clone_color: FAKECLONE1-clone allocation score on 18node3: 0
+clone_color: FAKECLONE1-clone allocation score on 18node4: 0
+clone_color: FAKECLONE1-clone allocation score on remote1: 0
+clone_color: FAKECLONE1:0 allocation score on 18builder: 0
+clone_color: FAKECLONE1:0 allocation score on 18node1: 0
+clone_color: FAKECLONE1:0 allocation score on 18node2: 0
+clone_color: FAKECLONE1:0 allocation score on 18node3: 0
+clone_color: FAKECLONE1:0 allocation score on 18node4: 0
+clone_color: FAKECLONE1:0 allocation score on remote1: 0
+clone_color: FAKECLONE1:1 allocation score on 18builder: 0
+clone_color: FAKECLONE1:1 allocation score on 18node1: 0
+clone_color: FAKECLONE1:1 allocation score on 18node2: 0
+clone_color: FAKECLONE1:1 allocation score on 18node3: 0
+clone_color: FAKECLONE1:1 allocation score on 18node4: 0
+clone_color: FAKECLONE1:1 allocation score on remote1: 0
+clone_color: FAKECLONE1:2 allocation score on 18builder: 0
+clone_color: FAKECLONE1:2 allocation score on 18node1: 0
+clone_color: FAKECLONE1:2 allocation score on 18node2: 0
+clone_color: FAKECLONE1:2 allocation score on 18node3: 0
+clone_color: FAKECLONE1:2 allocation score on 18node4: 0
+clone_color: FAKECLONE1:2 allocation score on remote1: 0
+clone_color: FAKECLONE1:3 allocation score on 18builder: 0
+clone_color: FAKECLONE1:3 allocation score on 18node1: 0
+clone_color: FAKECLONE1:3 allocation score on 18node2: 0
+clone_color: FAKECLONE1:3 allocation score on 18node3: 0
+clone_color: FAKECLONE1:3 allocation score on 18node4: 0
+clone_color: FAKECLONE1:3 allocation score on remote1: 0
+clone_color: FAKECLONE1:4 allocation score on 18builder: 0
+clone_color: FAKECLONE1:4 allocation score on 18node1: 0
+clone_color: FAKECLONE1:4 allocation score on 18node2: 0
+clone_color: FAKECLONE1:4 allocation score on 18node3: 0
+clone_color: FAKECLONE1:4 allocation score on 18node4: 0
+clone_color: FAKECLONE1:4 allocation score on remote1: 0
+clone_color: FAKECLONE1:5 allocation score on 18builder: 0
+clone_color: FAKECLONE1:5 allocation score on 18node1: 0
+clone_color: FAKECLONE1:5 allocation score on 18node2: 0
+clone_color: FAKECLONE1:5 allocation score on 18node3: 0
+clone_color: FAKECLONE1:5 allocation score on 18node4: 0
+clone_color: FAKECLONE1:5 allocation score on remote1: 0
+clone_color: FAKECLONE2-clone allocation score on 18builder: 0
+clone_color: FAKECLONE2-clone allocation score on 18node1: 0
+clone_color: FAKECLONE2-clone allocation score on 18node2: 0
+clone_color: FAKECLONE2-clone allocation score on 18node3: 0
+clone_color: FAKECLONE2-clone allocation score on 18node4: 0
+clone_color: FAKECLONE2-clone allocation score on remote1: 0
+clone_color: FAKECLONE2:0 allocation score on 18builder: 0
+clone_color: FAKECLONE2:0 allocation score on 18node1: 0
+clone_color: FAKECLONE2:0 allocation score on 18node2: 0
+clone_color: FAKECLONE2:0 allocation score on 18node3: 0
+clone_color: FAKECLONE2:0 allocation score on 18node4: 0
+clone_color: FAKECLONE2:0 allocation score on remote1: 0
+clone_color: FAKECLONE2:1 allocation score on 18builder: 0
+clone_color: FAKECLONE2:1 allocation score on 18node1: 0
+clone_color: FAKECLONE2:1 allocation score on 18node2: 0
+clone_color: FAKECLONE2:1 allocation score on 18node3: 0
+clone_color: FAKECLONE2:1 allocation score on 18node4: 0
+clone_color: FAKECLONE2:1 allocation score on remote1: 0
+clone_color: FAKECLONE2:2 allocation score on 18builder: 0
+clone_color: FAKECLONE2:2 allocation score on 18node1: 0
+clone_color: FAKECLONE2:2 allocation score on 18node2: 0
+clone_color: FAKECLONE2:2 allocation score on 18node3: 0
+clone_color: FAKECLONE2:2 allocation score on 18node4: 0
+clone_color: FAKECLONE2:2 allocation score on remote1: 0
+clone_color: FAKECLONE2:3 allocation score on 18builder: 0
+clone_color: FAKECLONE2:3 allocation score on 18node1: 0
+clone_color: FAKECLONE2:3 allocation score on 18node2: 0
+clone_color: FAKECLONE2:3 allocation score on 18node3: 0
+clone_color: FAKECLONE2:3 allocation score on 18node4: 0
+clone_color: FAKECLONE2:3 allocation score on remote1: 0
+clone_color: FAKECLONE2:4 allocation score on 18builder: 0
+clone_color: FAKECLONE2:4 allocation score on 18node1: 0
+clone_color: FAKECLONE2:4 allocation score on 18node2: 0
+clone_color: FAKECLONE2:4 allocation score on 18node3: 0
+clone_color: FAKECLONE2:4 allocation score on 18node4: 0
+clone_color: FAKECLONE2:4 allocation score on remote1: 0
+clone_color: FAKECLONE2:5 allocation score on 18builder: 0
+clone_color: FAKECLONE2:5 allocation score on 18node1: 0
+clone_color: FAKECLONE2:5 allocation score on 18node2: 0
+clone_color: FAKECLONE2:5 allocation score on 18node3: 0
+clone_color: FAKECLONE2:5 allocation score on 18node4: 0
+clone_color: FAKECLONE2:5 allocation score on remote1: 0
+native_color: FAKE1 allocation score on 18builder: 0
+native_color: FAKE1 allocation score on 18node1: 0
+native_color: FAKE1 allocation score on 18node2: 0
+native_color: FAKE1 allocation score on 18node3: 0
+native_color: FAKE1 allocation score on 18node4: 0
+native_color: FAKE1 allocation score on remote1: -INFINITY
+native_color: FAKE2 allocation score on 18builder: 0
+native_color: FAKE2 allocation score on 18node1: 0
+native_color: FAKE2 allocation score on 18node2: -INFINITY
+native_color: FAKE2 allocation score on 18node3: 0
+native_color: FAKE2 allocation score on 18node4: 0
+native_color: FAKE2 allocation score on remote1: 0
+native_color: FAKE3 allocation score on 18builder: 0
+native_color: FAKE3 allocation score on 18node1: 0
+native_color: FAKE3 allocation score on 18node2: 0
+native_color: FAKE3 allocation score on 18node3: INFINITY
+native_color: FAKE3 allocation score on 18node4: 0
+native_color: FAKE3 allocation score on remote1: 0
+native_color: FAKE4 allocation score on 18builder: 0
+native_color: FAKE4 allocation score on 18node1: 0
+native_color: FAKE4 allocation score on 18node2: 0
+native_color: FAKE4 allocation score on 18node3: 0
+native_color: FAKE4 allocation score on 18node4: 0
+native_color: FAKE4 allocation score on remote1: 0
+native_color: FAKE5 allocation score on 18builder: 0
+native_color: FAKE5 allocation score on 18node1: 0
+native_color: FAKE5 allocation score on 18node2: 0
+native_color: FAKE5 allocation score on 18node3: 0
+native_color: FAKE5 allocation score on 18node4: 0
+native_color: FAKE5 allocation score on remote1: 0
+native_color: FAKECLONE1:0 allocation score on 18builder: 0
+native_color: FAKECLONE1:0 allocation score on 18node1: 0
+native_color: FAKECLONE1:0 allocation score on 18node2: 0
+native_color: FAKECLONE1:0 allocation score on 18node3: 0
+native_color: FAKECLONE1:0 allocation score on 18node4: 0
+native_color: FAKECLONE1:0 allocation score on remote1: 0
+native_color: FAKECLONE1:1 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE1:1 allocation score on 18node1: 0
+native_color: FAKECLONE1:1 allocation score on 18node2: 0
+native_color: FAKECLONE1:1 allocation score on 18node3: 0
+native_color: FAKECLONE1:1 allocation score on 18node4: 0
+native_color: FAKECLONE1:1 allocation score on remote1: 0
+native_color: FAKECLONE1:2 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE1:2 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:2 allocation score on 18node2: 0
+native_color: FAKECLONE1:2 allocation score on 18node3: 0
+native_color: FAKECLONE1:2 allocation score on 18node4: 0
+native_color: FAKECLONE1:2 allocation score on remote1: 0
+native_color: FAKECLONE1:3 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE1:3 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:3 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE1:3 allocation score on 18node3: 0
+native_color: FAKECLONE1:3 allocation score on 18node4: 0
+native_color: FAKECLONE1:3 allocation score on remote1: 0
+native_color: FAKECLONE1:4 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE1:4 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:4 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE1:4 allocation score on 18node3: 0
+native_color: FAKECLONE1:4 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE1:4 allocation score on remote1: 0
+native_color: FAKECLONE1:5 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE1:5 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:5 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE1:5 allocation score on 18node3: 0
+native_color: FAKECLONE1:5 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE1:5 allocation score on remote1: -INFINITY
+native_color: FAKECLONE2:0 allocation score on 18builder: 0
+native_color: FAKECLONE2:0 allocation score on 18node1: 0
+native_color: FAKECLONE2:0 allocation score on 18node2: 0
+native_color: FAKECLONE2:0 allocation score on 18node3: 0
+native_color: FAKECLONE2:0 allocation score on 18node4: 0
+native_color: FAKECLONE2:0 allocation score on remote1: 0
+native_color: FAKECLONE2:1 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE2:1 allocation score on 18node1: 0
+native_color: FAKECLONE2:1 allocation score on 18node2: 0
+native_color: FAKECLONE2:1 allocation score on 18node3: 0
+native_color: FAKECLONE2:1 allocation score on 18node4: 0
+native_color: FAKECLONE2:1 allocation score on remote1: 0
+native_color: FAKECLONE2:2 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE2:2 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:2 allocation score on 18node2: 0
+native_color: FAKECLONE2:2 allocation score on 18node3: 0
+native_color: FAKECLONE2:2 allocation score on 18node4: 0
+native_color: FAKECLONE2:2 allocation score on remote1: 0
+native_color: FAKECLONE2:3 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE2:3 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:3 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE2:3 allocation score on 18node3: 0
+native_color: FAKECLONE2:3 allocation score on 18node4: 0
+native_color: FAKECLONE2:3 allocation score on remote1: 0
+native_color: FAKECLONE2:4 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE2:4 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:4 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE2:4 allocation score on 18node3: 0
+native_color: FAKECLONE2:4 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE2:4 allocation score on remote1: 0
+native_color: FAKECLONE2:5 allocation score on 18builder: -INFINITY
+native_color: FAKECLONE2:5 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:5 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE2:5 allocation score on 18node3: 0
+native_color: FAKECLONE2:5 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE2:5 allocation score on remote1: -INFINITY
+native_color: remote1 allocation score on 18builder: 0
+native_color: remote1 allocation score on 18node1: 0
+native_color: remote1 allocation score on 18node2: 0
+native_color: remote1 allocation score on 18node3: 0
+native_color: remote1 allocation score on 18node4: 0
+native_color: remote1 allocation score on remote1: -INFINITY
+native_color: shooter allocation score on 18builder: 0
+native_color: shooter allocation score on 18node1: 0
+native_color: shooter allocation score on 18node2: 0
+native_color: shooter allocation score on 18node3: 0
+native_color: shooter allocation score on 18node4: 0
+native_color: shooter allocation score on remote1: -INFINITY
diff --git a/pengine/test10/resource-discovery.summary b/pengine/test10/resource-discovery.summary
new file mode 100644
index 00000000000..af0e5b3e707
--- /dev/null
+++ b/pengine/test10/resource-discovery.summary
@@ -0,0 +1,124 @@
+
+Current cluster status:
+Online: [ 18builder 18node1 18node2 18node3 18node4 ]
+RemoteOFFLINE: [ remote1 ]
+
+ shooter	(stonith:fence_xvm):	Started 18node1 
+ remote1	(ocf::pacemaker:remote):	Stopped 
+ FAKE1	(ocf::heartbeat:Dummy):	Stopped 
+ FAKE2	(ocf::heartbeat:Dummy):	Started 18node2 
+ FAKE3	(ocf::heartbeat:Dummy):	Started 18builder 
+ FAKE4	(ocf::heartbeat:Dummy):	Started 18node1 
+ FAKE5	(ocf::heartbeat:Dummy):	Stopped 
+ Clone Set: FAKECLONE1-clone [FAKECLONE1]
+     Stopped: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+     Stopped: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+
+Transition Summary:
+ * Start   remote1	(18builder)
+ * Start   FAKE1	(18node2)
+ * Move    FAKE2	(Started 18node2 -> 18node3)
+ * Move    FAKE3	(Started 18builder -> 18node3)
+ * Move    FAKE4	(Started 18node1 -> 18node4)
+ * Start   FAKE5	(remote1)
+ * Start   FAKECLONE1:0	(18builder)
+ * Start   FAKECLONE1:1	(18node1)
+ * Start   FAKECLONE1:2	(18node2)
+ * Start   FAKECLONE1:3	(18node4)
+ * Start   FAKECLONE1:4	(remote1)
+ * Start   FAKECLONE1:5	(18node3)
+ * Start   FAKECLONE2:0	(18builder)
+ * Start   FAKECLONE2:1	(18node1)
+ * Start   FAKECLONE2:2	(18node2)
+ * Start   FAKECLONE2:3	(18node4)
+ * Start   FAKECLONE2:4	(remote1)
+ * Start   FAKECLONE2:5	(18node3)
+
+Executing cluster transition:
+ * Resource action: shooter         monitor on 18node4
+ * Resource action: shooter         monitor on 18node3
+ * Resource action: remote1         monitor on 18node4
+ * Resource action: remote1         monitor on 18node3
+ * Resource action: FAKE1           monitor on 18node4
+ * Resource action: FAKE1           monitor on 18node3
+ * Resource action: FAKE1           monitor on 18node2
+ * Resource action: FAKE1           monitor on 18node1
+ * Resource action: FAKE1           monitor on 18builder
+ * Resource action: FAKE3           monitor on 18node3
+ * Resource action: FAKE4           monitor on 18node4
+ * Resource action: FAKE5           monitor on 18node4
+ * Resource action: FAKE5           monitor on 18node3
+ * Resource action: FAKE5           monitor on 18node2
+ * Resource action: FAKE5           monitor on 18node1
+ * Resource action: FAKE5           monitor on 18builder
+ * Pseudo action:   FAKECLONE1-clone_start_0
+ * Resource action: FAKECLONE2:0    monitor on 18builder
+ * Resource action: FAKECLONE2:1    monitor on 18node1
+ * Resource action: FAKECLONE2:3    monitor on 18node4
+ * Resource action: FAKECLONE2:5    monitor on 18node3
+ * Pseudo action:   FAKECLONE2-clone_start_0
+ * Pseudo action:   probe_nodes_complete
+ * Resource action: remote1         start on 18builder
+ * Resource action: FAKE5           monitor on remote1
+ * Resource action: FAKECLONE1:4    monitor on remote1
+ * Resource action: FAKECLONE2:4    monitor on remote1
+ * Pseudo action:   probe_complete
+ * Resource action: remote1         monitor=60000 on 18builder
+ * Resource action: FAKE1           start on 18node2
+ * Resource action: FAKE2           stop on 18node2
+ * Resource action: FAKE3           stop on 18builder
+ * Resource action: FAKE4           stop on 18node1
+ * Resource action: FAKE5           start on remote1
+ * Resource action: FAKECLONE1:0    start on 18builder
+ * Resource action: FAKECLONE1:1    start on 18node1
+ * Resource action: FAKECLONE1:2    start on 18node2
+ * Resource action: FAKECLONE1:3    start on 18node4
+ * Resource action: FAKECLONE1:4    start on remote1
+ * Resource action: FAKECLONE1:5    start on 18node3
+ * Pseudo action:   FAKECLONE1-clone_running_0
+ * Resource action: FAKECLONE2:0    start on 18builder
+ * Resource action: FAKECLONE2:1    start on 18node1
+ * Resource action: FAKECLONE2:2    start on 18node2
+ * Resource action: FAKECLONE2:3    start on 18node4
+ * Resource action: FAKECLONE2:4    start on remote1
+ * Resource action: FAKECLONE2:5    start on 18node3
+ * Pseudo action:   FAKECLONE2-clone_running_0
+ * Pseudo action:   all_stopped
+ * Resource action: FAKE1           monitor=60000 on 18node2
+ * Resource action: FAKE2           start on 18node3
+ * Resource action: FAKE3           start on 18node3
+ * Resource action: FAKE4           start on 18node4
+ * Resource action: FAKE5           monitor=60000 on remote1
+ * Resource action: FAKECLONE1:0    monitor=60000 on 18builder
+ * Resource action: FAKECLONE1:1    monitor=60000 on 18node1
+ * Resource action: FAKECLONE1:2    monitor=60000 on 18node2
+ * Resource action: FAKECLONE1:3    monitor=60000 on 18node4
+ * Resource action: FAKECLONE1:4    monitor=60000 on remote1
+ * Resource action: FAKECLONE1:5    monitor=60000 on 18node3
+ * Resource action: FAKECLONE2:0    monitor=60000 on 18builder
+ * Resource action: FAKECLONE2:1    monitor=60000 on 18node1
+ * Resource action: FAKECLONE2:2    monitor=60000 on 18node2
+ * Resource action: FAKECLONE2:3    monitor=60000 on 18node4
+ * Resource action: FAKECLONE2:4    monitor=60000 on remote1
+ * Resource action: FAKECLONE2:5    monitor=60000 on 18node3
+ * Resource action: FAKE2           monitor=60000 on 18node3
+ * Resource action: FAKE3           monitor=60000 on 18node3
+ * Resource action: FAKE4           monitor=60000 on 18node4
+
+Revised cluster status:
+Online: [ 18builder 18node1 18node2 18node3 18node4 ]
+RemoteOnline: [ remote1 ]
+
+ shooter	(stonith:fence_xvm):	Started 18node1 
+ remote1	(ocf::pacemaker:remote):	Started 18builder 
+ FAKE1	(ocf::heartbeat:Dummy):	Started 18node2 
+ FAKE2	(ocf::heartbeat:Dummy):	Started 18node3 
+ FAKE3	(ocf::heartbeat:Dummy):	Started 18node3 
+ FAKE4	(ocf::heartbeat:Dummy):	Started 18node4 
+ FAKE5	(ocf::heartbeat:Dummy):	Started remote1 
+ Clone Set: FAKECLONE1-clone [FAKECLONE1]
+     Started: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+     Started: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+
diff --git a/pengine/test10/resource-discovery.xml b/pengine/test10/resource-discovery.xml
new file mode 100644
index 00000000000..5836804b8af
--- /dev/null
+++ b/pengine/test10/resource-discovery.xml
@@ -0,0 +1,185 @@
+<cib epoch="11" num_updates="1" admin_epoch="0" validate-with="pacemaker-next" cib-last-written="Tue Sep  3 17:59:24 2013" update-origin="18builder" update-client="crm_resource" crm_feature_set="3.0.7" have-quorum="1" dc-uuid="5">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-ab50afe"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="18node1"/>
+      <node id="2" uname="18node2"/>
+      <node id="3" uname="18node3"/>
+      <node id="4" uname="18node4"/>
+      <node id="5" uname="18builder"/>
+    </nodes>
+    <resources>
+      <primitive class="stonith" id="shooter" type="fence_xvm">
+        <instance_attributes id="shooter-instance_attributes"/>
+        <operations>
+          <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="remote1" provider="pacemaker" type="remote">
+        <instance_attributes id="remote1-instance_attributes"/>
+        <operations>
+          <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+        <meta_attributes id="remote1-meta_attributes"/>
+      </primitive>
+      <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE1-instance_attributes"/>
+        <meta_attributes id="FAKE1-meta_attributes">
+        </meta_attributes>
+        <operations>
+          <op id="FAKE1-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE2-instance_attributes"/>
+        <meta_attributes id="FAKE2-meta_attributes">
+        </meta_attributes>
+        <operations>
+          <op id="FAKE2-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE3-instance_attributes"/>
+        <operations>
+          <op id="FAKE3-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE4" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE4-instance_attributes"/>
+        <operations>
+          <op id="FAKE4-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE5" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE5-instance_attributes"/>
+        <operations>
+          <op id="FAKE5-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <clone id="FAKECLONE1-clone">
+        <primitive class="ocf" id="FAKECLONE1" provider="heartbeat" type="Dummy">
+          <instance_attributes id="FAKECLONE1-instance_attributes"/>
+          <operations>
+            <op id="FAKECLONE1-monitor-interval-60s" interval="60s" name="monitor"/>
+          </operations>
+        </primitive>
+      </clone>
+      <clone id="FAKECLONE2-clone">
+        <primitive class="ocf" id="FAKECLONE2" provider="heartbeat" type="Dummy">
+          <instance_attributes id="FAKECLONE2-instance_attributes"/>
+          <operations>
+            <op id="FAKECLONE2-monitor-interval-60s" interval="60s" name="monitor"/>
+          </operations>
+        </primitive>
+      </clone>
+    </resources>
+    <constraints>
+      <rsc_location id="location-FAKE1-remote1--INFINITY" node="remote1" rsc="FAKE1" score="-INFINITY" resource-discovery="never" />
+      <rsc_location id="location-FAKE2-18node2--INFINITY" node="18node2" rsc="FAKE2" score="-INFINITY" resource-discovery="exclusive" />
+      <rsc_location id="location-FAKE3-18node3--INFINITY" node="18node3" rsc="FAKE3" score="INFINITY" resource-discovery="exclusive" />
+      <rsc_location id="location-FAKE4-1-18node3--INFINITY" node="18node4" rsc="FAKE4" score="0" resource-discovery="always" />
+      <rsc_location id="location-FAKE4-2-18node3--INFINITY" node="18node4" rsc="FAKE4" score="0" resource-discovery="never" />
+      <rsc_location id="location-FAKE4-3-18node3--INFINITY" node="18node4" rsc="FAKE4" score="0" resource-discovery="exclusive" />
+      <rsc_location id="location-FAKE5-18node3--INFINITY" node="18node4" rsc="FAKE5" score="0" resource-discovery="always" />
+      <rsc_location id="location-FAKECLONE1-remote--INFINITY" node="remote1" rsc="FAKECLONE1-clone" score="0" resource-discovery="exclusive" />
+      <rsc_location id="location-FAKECLONE2-18node2--INFINITY" node="18node2" rsc="FAKECLONE2-clone" score="0" resource-discovery="never" />
+    </constraints>
+  </configuration>
+  <status>
+    <node_state id="5" uname="18builder" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <transient_attributes id="5">
+        <instance_attributes id="status-5">
+        </instance_attributes>
+      </transient_attributes>
+      <lrm id="5">
+        <lrm_resources>
+          <lrm_resource id="shooter" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="shooter_last_0" operation_key="shooter_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="11:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="16" rc-code="0" op-status="0" interval="0" last-run="1378248545" last-rc-change="1378248545" exec-time="1" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:3:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:3:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248534" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="remote1_last_0" operation_key="remote1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:6:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:6:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="13" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="remote1_monitor_60000" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:5:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:5:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="3" rc-code="0" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="7:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;7:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="39" rc-code="7" op-status="0" interval="0" last-run="1378249145" last-rc-change="1378249145" exec-time="61" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="18:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;18:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1378249147" last-rc-change="1378249147" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE3_monitor_60000" operation_key="FAKE3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="19:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="56" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249147" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1378249149" last-rc-change="1378249149" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="1" uname="18node1" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="shooter" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1378248547" last-rc-change="1378248547" exec-time="19" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="12" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248548" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="25" rc-code="7" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE3_last_failure_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="32" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="15" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="22:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="36" queue-time="38" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE4_monitor_60000" operation_key="FAKE4_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="23:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="44" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249148" exec-time="14" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="1">
+        <instance_attributes id="status-1">
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="2" uname="18node2" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
+      <lrm id="2">
+        <lrm_resources>
+          <lrm_resource id="shooter" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="980" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;10:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="16:9:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;16:9:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="18" queue-time="9" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE2_monitor_60000" operation_key="FAKE2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:9:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:9:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="22" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249144" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="27" rc-code="7" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;13:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="2">
+        <instance_attributes id="status-2">
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="3" uname="18node3" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member"/>
+    <node_state id="4" uname="18node4" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member"/>
+    <node_state remote_node="true" id="remote1" uname="remote1" crm-debug-origin="remote_init_cib_status">
+      <transient_attributes id="remote1">
+        <instance_attributes id="status-remote1">
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+  </status>
+</cib>

From dc7fe545d98b01ffc95bca0a5a6d6ff0732d14e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kristoffer=20Gr=C3=B6nlund?= <krig@koru.se>
Date: Thu, 2 Oct 2014 15:19:41 +0200
Subject: [PATCH 05/29] Fix: ping: Correctly advertise multiplier default

Advertise the multiplier default value in the meta data.
---
 extra/resources/ping | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extra/resources/ping b/extra/resources/ping
index b9a69b829bb..c525b41297a 100755
--- a/extra/resources/ping
+++ b/extra/resources/ping
@@ -77,7 +77,7 @@ The name of the attributes to set.  This is the name to be used in the constrain
 The number by which to multiply the number of connected ping nodes by
 </longdesc>
 <shortdesc lang="en">Value multiplier</shortdesc>
-<content type="integer" default=""/>
+<content type="integer" default="1"/>
 </parameter>
 
 <parameter name="host_list" unique="0" required="1">

From e3277d0ec52d70921f5f58d443c4d28f72b8aac9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kristoffer=20Gr=C3=B6nlund?= <krig@koru.se>
Date: Thu, 2 Oct 2014 15:38:10 +0200
Subject: [PATCH 06/29] Low: ping: Add use_fping parameter

Add parameter that allows control over which ping
binary is used. If set to 0, fping will not be used,
instead ping is used even if fping is available.
---
 extra/resources/ping | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/extra/resources/ping b/extra/resources/ping
index c525b41297a..09c72d9f1cb 100755
--- a/extra/resources/ping
+++ b/extra/resources/ping
@@ -121,6 +121,15 @@ Default never fails.
 <content type="integer" default=""/>
 </parameter>
 
+<parameter name="use_fping" unique="0">
+<longdesc lang="en">
+Use fping rather than ping, if found. If set to 0, fping
+will not be used even if present.
+</longdesc>
+<shortdesc lang="en">Use fping if available</shortdesc>
+<content type="boolean" default="1"/>
+</parameter>
+
 <parameter name="debug" unique="0">
 <longdesc lang="en">
 Enables to use default attrd_updater verbose logging on every call.
@@ -274,7 +283,7 @@ ping_check() {
 
 ping_update() {
     
-    if have_binary fping; then
+    if ocf_is_true "$OCF_RESKEY_use_fping" && have_binary fping; then
 	fping_check
 	active=$?
     else
@@ -306,6 +315,7 @@ ping_update() {
 : ${OCF_RESKEY_multiplier:="1"}
 : ${OCF_RESKEY_debug:="false"}
 : ${OCF_RESKEY_failure_score:="0"}
+: ${OCF_RESKEY_use_fping:="1"}
 
 : ${OCF_RESKEY_CRM_meta_timeout:="20000"}
 : ${OCF_RESKEY_CRM_meta_globally_unique:="true"}

From 1819b40055f2a148c484c1a20b2668371465a18e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kristoffer=20Gr=C3=B6nlund?= <krig@koru.se>
Date: Thu, 2 Oct 2014 15:52:51 +0200
Subject: [PATCH 07/29] Low: ping: Remove incorrectly advertised
 migrate_to|migrate_from

---
 extra/resources/ping | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extra/resources/ping b/extra/resources/ping
index 09c72d9f1cb..fcd1a920acb 100755
--- a/extra/resources/ping
+++ b/extra/resources/ping
@@ -163,7 +163,7 @@ ping_conditional_log() {
 
 ping_usage() {
 	cat <<END
-usage: $0 {start|stop|monitor|migrate_to|migrate_from|validate-all|meta-data}
+usage: $0 {start|stop|monitor|validate-all|meta-data}
 
 Expects to have a fully populated OCF RA-compliant environment set.
 END

From 77de4f36b257a1509039944cfdbcc85bf291a101 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kristoffer=20Gr=C3=B6nlund?= <krig@koru.se>
Date: Thu, 2 Oct 2014 15:56:59 +0200
Subject: [PATCH 08/29] Low: ping: Pass extra options to fping

---
 extra/resources/ping | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extra/resources/ping b/extra/resources/ping
index fcd1a920acb..e977e6b684c 100755
--- a/extra/resources/ping
+++ b/extra/resources/ping
@@ -234,7 +234,7 @@ fping_check() {
     n=$OCF_RESKEY_attempts
     timeout=`expr $OCF_RESKEY_timeout \* 1000 / $OCF_RESKEY_attempts`
 
-    cmd="fping -r $OCF_RESKEY_attempts -t $timeout -B 1.0 $OCF_RESKEY_host_list"
+    cmd="fping -r $OCF_RESKEY_attempts -t $timeout -B 1.0 $OCF_RESKEY_options $OCF_RESKEY_host_list"
     $cmd>$f_out 2>$f_err; rc=$?
     active=`grep alive $f_out|wc -l`
 

From 27f8047c0afa77de968fd597b64fc1895b6444a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kristoffer=20Gr=C3=B6nlund?= <krig@koru.se>
Date: Thu, 2 Oct 2014 15:59:09 +0200
Subject: [PATCH 09/29] Low: ping: Correct metadata for attempts parameter

---
 extra/resources/ping | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extra/resources/ping b/extra/resources/ping
index e977e6b684c..e7b9973681b 100755
--- a/extra/resources/ping
+++ b/extra/resources/ping
@@ -93,7 +93,7 @@ The list of ping nodes to count.
 Number of ping attempts, per host, before declaring it dead
 </longdesc>
 <shortdesc lang="en">no. of ping attempts</shortdesc>
-<content type="integer" default="2"/>
+<content type="integer" default="3"/>
 </parameter>
 
 <parameter name="timeout" unique="0">

From c747f20d19830ed6deb1ae97b6415c195569e0fa Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Thu, 2 Oct 2014 17:23:27 +0200
Subject: [PATCH 10/29] mainloop: fix potential segfault for fd = 0

Some monitoring software may explicitly *close* stdin,
(not just redirect from /dev/null).

That led to segfault, because the next allocated new fd would be 0,
mainloop_add_fd would return NULL (and the caller would not check).

Reproduce:
	crm_mon -1 <&-

Since 0 is a legal value for fd, allow it in mainloop_add_fd, too.
---
 lib/common/mainloop.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
index ac395ecf3a3..f2295ff439c 100644
--- a/lib/common/mainloop.c
+++ b/lib/common/mainloop.c
@@ -799,7 +799,7 @@ mainloop_add_fd(const char *name, int priority, int fd, void *userdata,
 {
     mainloop_io_t *client = NULL;
 
-    if (fd > 0) {
+    if (fd >= 0) {
         client = calloc(1, sizeof(mainloop_io_t));
         client->name = strdup(name);
         client->userdata = userdata;

From ecdfa39ac2511c470b046942d0d49a9f7f6f662a Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Thu, 2 Oct 2014 13:19:43 -0400
Subject: [PATCH 11/29] High: pengine: per-node control over resource discovery

By using the 'resource-discovery-enabled' node attribute, users
can disable/enable resource discovery (probes) on a per node
basis. By default, resource discovery is enabled still. No default
behavior has changed with the introduction of this option.
---
 include/crm/msg_xml.h        |  1 +
 include/crm/pengine/status.h |  2 ++
 lib/pengine/unpack.c         | 15 +++++++++++++++
 pengine/allocate.c           |  4 ++++
 4 files changed, 22 insertions(+)

diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
index 8e5708cf85d..64c971d7545 100644
--- a/include/crm/msg_xml.h
+++ b/include/crm/msg_xml.h
@@ -323,6 +323,7 @@
 #  define XML_NVPAIR_ATTR_VALUE        	"value"
 
 #  define XML_NODE_ATTR_STATE		"state"
+#  define XML_NODE_ATTR_RSC_DISCOVERY   "resource-discovery-enabled"
 
 #  define XML_CONFIG_ATTR_DC_DEADTIME	"dc-deadtime"
 #  define XML_CONFIG_ATTR_ELECTION_FAIL	"election-timeout"
diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
index 3708d7ef164..f08a910635a 100644
--- a/include/crm/pengine/status.h
+++ b/include/crm/pengine/status.h
@@ -136,6 +136,8 @@ struct node_shared_s {
     gboolean shutdown;
     gboolean expected_up;
     gboolean is_dc;
+    gboolean rsc_discovery_enabled;
+
     int num_resources;
     GListPtr running_rsc;       /* resource_t* */
     GListPtr allocated_rsc;     /* resource_t* */
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 7127c12795f..395782ae2e0 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -294,6 +294,7 @@ create_node(const char *id, const char *uname, const char *type, const char *sco
     new_node->details->uname = uname;
     new_node->details->online = FALSE;
     new_node->details->shutdown = FALSE;
+    new_node->details->rsc_discovery_enabled = TRUE;
     new_node->details->running_rsc = NULL;
     new_node->details->type = node_ping;
 
@@ -988,6 +989,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set)
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
             xmlNode *attrs = NULL;
+            const char *resource_discovery_enabled = NULL;
 
             id = crm_element_value(state, XML_ATTR_ID);
             uname = crm_element_value(state, XML_ATTR_UNAME);
@@ -1027,6 +1029,12 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set)
                 this_node->details->maintenance = TRUE;
             }
 
+            resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
+            if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
+                crm_info("Node %s has resource discovery disabled", this_node->details->uname);
+                this_node->details->rsc_discovery_enabled = FALSE;
+            }
+
             crm_trace("determining node state");
             determine_online_status(state, this_node, data_set);
 
@@ -1102,6 +1110,7 @@ unpack_remote_status(xmlNode * status, pe_working_set_t * data_set)
 
     /* process attributes */
     for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) {
+        const char *resource_discovery_enabled = NULL;
         xmlNode *attrs = NULL;
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
             continue;
@@ -1125,6 +1134,12 @@ unpack_remote_status(xmlNode * status, pe_working_set_t * data_set)
             crm_info("Node %s is in standby-mode", this_node->details->uname);
             this_node->details->standby = TRUE;
         }
+
+        resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
+        if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
+            crm_info("Node %s has resource discovery disabled", this_node->details->uname);
+            this_node->details->rsc_discovery_enabled = FALSE;
+        }
     }
 
     /* process node rsc status */
diff --git a/pengine/allocate.c b/pengine/allocate.c
index 57d81ef451f..e708e265f79 100644
--- a/pengine/allocate.c
+++ b/pengine/allocate.c
@@ -861,6 +861,10 @@ probe_resources(pe_working_set_t * data_set)
             /* TODO enable container node probes once ordered probing is implemented. */
             continue;
 
+        } else if (node->details->rsc_discovery_enabled == FALSE) {
+            /* resource discovery is disabled for this node */
+            continue;
+
         } else if (probe_complete == NULL) {
             probe_complete = get_pseudo_op(CRM_OP_PROBED, data_set);
             if (is_set(data_set->flags, pe_flag_have_remote_nodes)) {

From 5ecc36c16617e15292401d42363d78ae080bba22 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Thu, 2 Oct 2014 13:21:06 -0400
Subject: [PATCH 12/29] High: crm_resource: avoid deletion of lrm cache on node
 with resource discovery disabled.

This patch prevents users from deleting lrm cache on nodes that
have resource discovery disabled. Deleting the lrm cache in this
situation would result a failure to re-detect the resources currently
running on the node. Instead, lrm delete is skipped, but failcounts
are still cleaned up properly.
---
 tools/crm_resource.c | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 56583e0df08..e2da2442693 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -94,6 +94,10 @@ resource_ipc_connection_destroy(gpointer user_data)
 static void
 start_mainloop(void)
 {
+    if (crmd_replies_needed == 0) {
+        return;
+    }
+
     mainloop = g_main_new(FALSE);
     fprintf(stderr, "Waiting for %d replies from the CRMd", crmd_replies_needed);
     crm_debug("Waiting for %d replies from the CRMd", crmd_replies_needed);
@@ -789,6 +793,7 @@ delete_lrm_rsc(cib_t *cib_conn, crm_ipc_t * crmd_channel, const char *host_uname
                resource_t * rsc, pe_working_set_t * data_set)
 {
     int rc = pcmk_ok;
+    node_t *node = NULL;
 
     if (rsc == NULL) {
         return -ENXIO;
@@ -807,7 +812,7 @@ delete_lrm_rsc(cib_t *cib_conn, crm_ipc_t * crmd_channel, const char *host_uname
         GListPtr lpc = NULL;
 
         for (lpc = data_set->nodes; lpc != NULL; lpc = lpc->next) {
-            node_t *node = (node_t *) lpc->data;
+            node = (node_t *) lpc->data;
 
             if (node->details->online) {
                 delete_lrm_rsc(cib_conn, crmd_channel, node->details->uname, rsc, data_set);
@@ -817,15 +822,20 @@ delete_lrm_rsc(cib_t *cib_conn, crm_ipc_t * crmd_channel, const char *host_uname
         return pcmk_ok;
     }
 
-    printf("Cleaning up %s on %s\n", rsc->id, host_uname);
-    rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
+    node = pe_find_node(data_set->nodes, host_uname);
+
+    if (node && node->details->rsc_discovery_enabled) {
+        printf("Cleaning up %s on %s\n", rsc->id, host_uname);
+        rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
+    } else {
+        printf("Resource discovery disabled on %s. Unable to delete lrm state.\n", host_uname);
+    }
 
     if (rc == pcmk_ok) {
         char *attr_name = NULL;
         const char *id = rsc->id;
-        node_t *node = pe_find_node(data_set->nodes, host_uname);
 
-        if(node && node->details->remote_rsc == NULL) {
+        if(node && node->details->remote_rsc == NULL && node->details->rsc_discovery_enabled) {
             crmd_replies_needed++;
         }
         if (rsc->clone_name) {

From 02060d234e07f4daed53d5e1447d4ecf46dfc649 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Thu, 2 Oct 2014 13:43:28 -0400
Subject: [PATCH 13/29] Low: pengine: regression test for pengine per node rsc
 discovery

---
 pengine/regression.sh                         |   3 +-
 pengine/test10/rsc-discovery-per-node.dot     | 175 ++++
 pengine/test10/rsc-discovery-per-node.exp     | 983 ++++++++++++++++++
 pengine/test10/rsc-discovery-per-node.scores  | 199 ++++
 pengine/test10/rsc-discovery-per-node.summary | 123 +++
 pengine/test10/rsc-discovery-per-node.xml     | 182 ++++
 6 files changed, 1664 insertions(+), 1 deletion(-)
 create mode 100644 pengine/test10/rsc-discovery-per-node.dot
 create mode 100644 pengine/test10/rsc-discovery-per-node.exp
 create mode 100644 pengine/test10/rsc-discovery-per-node.scores
 create mode 100644 pengine/test10/rsc-discovery-per-node.summary
 create mode 100644 pengine/test10/rsc-discovery-per-node.xml

diff --git a/pengine/regression.sh b/pengine/regression.sh
index 8acf890fea9..a9a56059cf2 100755
--- a/pengine/regression.sh
+++ b/pengine/regression.sh
@@ -770,7 +770,8 @@ do_test remote-recover         "Recover connection resource after cluster-node f
 do_test remote-stale-node-entry "Make sure we properly handle leftover remote-node entries in the node section"
 
 echo ""
-do_test resource-discovery "Exercises resource-discovery location constraint option."
+do_test resource-discovery      "Exercises resource-discovery location constraint option."
+do_test rsc-discovery-per-node  "Disable resource discovery per node"
 
 echo ""
 test_results
diff --git a/pengine/test10/rsc-discovery-per-node.dot b/pengine/test10/rsc-discovery-per-node.dot
new file mode 100644
index 00000000000..4ddaba10e04
--- /dev/null
+++ b/pengine/test10/rsc-discovery-per-node.dot
@@ -0,0 +1,175 @@
+ digraph "g" {
+"FAKE1_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
+"FAKE1_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE1_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE1_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE1_start_0 18node2" -> "FAKE1_monitor_60000 18node2" [ style = bold]
+"FAKE1_start_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE2_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE2_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE2_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE2_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE2_monitor_60000 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE2_start_0 18node3" -> "FAKE2_monitor_60000 18node3" [ style = bold]
+"FAKE2_start_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE2_stop_0 18node2" -> "FAKE2_start_0 18node3" [ style = bold]
+"FAKE2_stop_0 18node2" -> "all_stopped" [ style = bold]
+"FAKE2_stop_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKE3_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE3_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE3_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE3_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE3_monitor_60000 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE3_start_0 18node4" -> "FAKE3_monitor_60000 18node4" [ style = bold]
+"FAKE3_start_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE3_stop_0 18builder" -> "FAKE3_start_0 18node4" [ style = bold]
+"FAKE3_stop_0 18builder" -> "all_stopped" [ style = bold]
+"FAKE3_stop_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE4_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE4_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE4_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE4_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE4_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKE4_start_0 remote1" -> "FAKE4_monitor_60000 remote1" [ style = bold]
+"FAKE4_start_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKE4_stop_0 18node1" -> "FAKE4_start_0 remote1" [ style = bold]
+"FAKE4_stop_0 18node1" -> "all_stopped" [ style = bold]
+"FAKE4_stop_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
+"FAKE5_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKE5_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKE5_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE5_start_0 18builder" -> "FAKE5_monitor_60000 18builder" [ style = bold]
+"FAKE5_start_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:0_start_0 18node1" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:1_start_0 18node2" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:2_start_0 18node3" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:3_start_0 18node4" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:4_start_0 remote1" [ style = bold]
+"FAKECLONE1-clone_start_0" -> "FAKECLONE1:5_start_0 18builder" [ style = bold]
+"FAKECLONE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE1:0_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:0_start_0 18node1" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:0_start_0 18node1" -> "FAKECLONE1:0_monitor_60000 18node1" [ style = bold]
+"FAKECLONE1:0_start_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:1_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:1_start_0 18node2" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:1_start_0 18node2" -> "FAKECLONE1:1_monitor_60000 18node2" [ style = bold]
+"FAKECLONE1:1_start_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:2_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKECLONE1:2_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:2_monitor_60000 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:2_start_0 18node3" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:2_start_0 18node3" -> "FAKECLONE1:2_monitor_60000 18node3" [ style = bold]
+"FAKECLONE1:2_start_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:3_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKECLONE1:3_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:3_monitor_60000 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:3_start_0 18node4" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:3_start_0 18node4" -> "FAKECLONE1:3_monitor_60000 18node4" [ style = bold]
+"FAKECLONE1:3_start_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:4_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:4_start_0 remote1" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:4_start_0 remote1" -> "FAKECLONE1:4_monitor_60000 remote1" [ style = bold]
+"FAKECLONE1:4_start_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:5_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
+"FAKECLONE1:5_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:5_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:5_start_0 18builder" -> "FAKECLONE1-clone_running_0" [ style = bold]
+"FAKECLONE1:5_start_0 18builder" -> "FAKECLONE1:5_monitor_60000 18builder" [ style = bold]
+"FAKECLONE1:5_start_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:0_start_0 18node1" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:1_start_0 18node2" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:2_start_0 18node3" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:3_start_0 18node4" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:4_start_0 remote1" [ style = bold]
+"FAKECLONE2-clone_start_0" -> "FAKECLONE2:5_start_0 18builder" [ style = bold]
+"FAKECLONE2-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2:0_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:0_start_0 18node1" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:0_start_0 18node1" -> "FAKECLONE2:0_monitor_60000 18node1" [ style = bold]
+"FAKECLONE2:0_start_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:1_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:1_start_0 18node2" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:1_start_0 18node2" -> "FAKECLONE2:1_monitor_60000 18node2" [ style = bold]
+"FAKECLONE2:1_start_0 18node2" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:2_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"FAKECLONE2:2_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:2_monitor_60000 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:2_start_0 18node3" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:2_start_0 18node3" -> "FAKECLONE2:2_monitor_60000 18node3" [ style = bold]
+"FAKECLONE2:2_start_0 18node3" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:3_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"FAKECLONE2:3_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:3_monitor_60000 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:3_start_0 18node4" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:3_start_0 18node4" -> "FAKECLONE2:3_monitor_60000 18node4" [ style = bold]
+"FAKECLONE2:3_start_0 18node4" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:4_monitor_60000 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:4_start_0 remote1" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:4_start_0 remote1" -> "FAKECLONE2:4_monitor_60000 remote1" [ style = bold]
+"FAKECLONE2:4_start_0 remote1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:5_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
+"FAKECLONE2:5_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:5_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:5_start_0 18builder" -> "FAKECLONE2-clone_running_0" [ style = bold]
+"FAKECLONE2:5_start_0 18builder" -> "FAKECLONE2:5_monitor_60000 18builder" [ style = bold]
+"FAKECLONE2:5_start_0 18builder" [ style=bold color="green" fontcolor="black"]
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"probe_complete 18builder" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18builder" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node3" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node3" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node4" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node4" [ style=bold color="green" fontcolor="black"]
+"probe_complete" -> "FAKE1_start_0 18node2" [ style = bold]
+"probe_complete" -> "FAKE2_start_0 18node3" [ style = bold]
+"probe_complete" -> "FAKE2_stop_0 18node2" [ style = bold]
+"probe_complete" -> "FAKE3_start_0 18node4" [ style = bold]
+"probe_complete" -> "FAKE3_stop_0 18builder" [ style = bold]
+"probe_complete" -> "FAKE4_start_0 remote1" [ style = bold]
+"probe_complete" -> "FAKE4_stop_0 18node1" [ style = bold]
+"probe_complete" -> "FAKE5_start_0 18builder" [ style = bold]
+"probe_complete" -> "FAKECLONE1:0_start_0 18node1" [ style = bold]
+"probe_complete" -> "FAKECLONE1:1_start_0 18node2" [ style = bold]
+"probe_complete" -> "FAKECLONE1:2_start_0 18node3" [ style = bold]
+"probe_complete" -> "FAKECLONE1:3_start_0 18node4" [ style = bold]
+"probe_complete" -> "FAKECLONE1:4_start_0 remote1" [ style = bold]
+"probe_complete" -> "FAKECLONE1:5_start_0 18builder" [ style = bold]
+"probe_complete" -> "FAKECLONE2:0_start_0 18node1" [ style = bold]
+"probe_complete" -> "FAKECLONE2:1_start_0 18node2" [ style = bold]
+"probe_complete" -> "FAKECLONE2:2_start_0 18node3" [ style = bold]
+"probe_complete" -> "FAKECLONE2:3_start_0 18node4" [ style = bold]
+"probe_complete" -> "FAKECLONE2:4_start_0 remote1" [ style = bold]
+"probe_complete" -> "FAKECLONE2:5_start_0 18builder" [ style = bold]
+"probe_complete" [ style=bold color="green" fontcolor="orange"]
+"probe_nodes_complete" -> "remote1_start_0 18builder" [ style = bold]
+"probe_nodes_complete" [ style=bold color="green" fontcolor="orange"]
+"remote1_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"remote1_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"remote1_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"remote1_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+"remote1_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"]
+"remote1_start_0 18builder" -> "FAKE4_monitor_60000 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKE4_start_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE1:4_monitor_60000 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE1:4_start_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE2:4_monitor_60000 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "FAKECLONE2:4_start_0 remote1" [ style = bold]
+"remote1_start_0 18builder" -> "remote1_monitor_60000 18builder" [ style = bold]
+"remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"]
+"shooter_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
+"shooter_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
+"shooter_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
+"shooter_monitor_0 18node4" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/rsc-discovery-per-node.exp b/pengine/test10/rsc-discovery-per-node.exp
new file mode 100644
index 00000000000..0ffd3a1c399
--- /dev/null
+++ b/pengine/test10/rsc-discovery-per-node.exp
@@ -0,0 +1,983 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="24" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="shooter" class="stonith" type="fence_xvm"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="14" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="shooter" class="stonith" type="fence_xvm"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="2">
+    <action_set>
+      <rsc_op id="36" operation="monitor" operation_key="remote1_monitor_60000" on_node="18builder" on_node_uuid="5">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="3">
+    <action_set>
+      <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="7" operation="probe_nodes_complete" operation_key="probe_nodes_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="4">
+    <action_set>
+      <rsc_op id="25" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="5">
+    <action_set>
+      <rsc_op id="15" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="6">
+    <action_set>
+      <rsc_op id="38" operation="monitor" operation_key="FAKE1_monitor_60000" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="37" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="7">
+    <action_set>
+      <rsc_op id="37" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="8">
+    <action_set>
+      <rsc_op id="26" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="9">
+    <action_set>
+      <rsc_op id="16" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="10">
+    <action_set>
+      <rsc_op id="9" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="11">
+    <action_set>
+      <rsc_op id="41" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="40" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="12">
+    <action_set>
+      <rsc_op id="40" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="39" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="13">
+    <action_set>
+      <rsc_op id="39" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="14">
+    <action_set>
+      <rsc_op id="27" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="15">
+    <action_set>
+      <rsc_op id="17" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="16">
+    <action_set>
+      <rsc_op id="44" operation="monitor" operation_key="FAKE3_monitor_60000" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="43" operation="start" operation_key="FAKE3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="17">
+    <action_set>
+      <rsc_op id="43" operation="start" operation_key="FAKE3_start_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="42" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="18">
+    <action_set>
+      <rsc_op id="42" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="19">
+    <action_set>
+      <rsc_op id="28" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="20">
+    <action_set>
+      <rsc_op id="18" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="21">
+    <action_set>
+      <rsc_op id="47" operation="monitor" operation_key="FAKE4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="46" operation="start" operation_key="FAKE4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="22">
+    <action_set>
+      <rsc_op id="46" operation="start" operation_key="FAKE4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="45" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="23">
+    <action_set>
+      <rsc_op id="45" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="24">
+    <action_set>
+      <rsc_op id="29" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="25">
+    <action_set>
+      <rsc_op id="19" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="26">
+    <action_set>
+      <rsc_op id="49" operation="monitor" operation_key="FAKE5_monitor_60000" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="48" operation="start" operation_key="FAKE5_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="27">
+    <action_set>
+      <rsc_op id="48" operation="start" operation_key="FAKE5_start_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="28">
+    <action_set>
+      <rsc_op id="30" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="29">
+    <action_set>
+      <rsc_op id="20" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="30">
+    <action_set>
+      <rsc_op id="10" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="31">
+    <action_set>
+      <rsc_op id="51" operation="monitor" operation_key="FAKECLONE1:0_monitor_60000" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="50" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="32">
+    <action_set>
+      <rsc_op id="50" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="33">
+    <action_set>
+      <rsc_op id="53" operation="monitor" operation_key="FAKECLONE1:1_monitor_60000" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="52" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="34">
+    <action_set>
+      <rsc_op id="52" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="35">
+    <action_set>
+      <rsc_op id="55" operation="monitor" operation_key="FAKECLONE1:2_monitor_60000" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="54" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="36">
+    <action_set>
+      <rsc_op id="54" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="37">
+    <action_set>
+      <rsc_op id="21" operation="monitor" operation_key="FAKECLONE1:2_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="38">
+    <action_set>
+      <rsc_op id="57" operation="monitor" operation_key="FAKECLONE1:3_monitor_60000" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="56" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="39">
+    <action_set>
+      <rsc_op id="56" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="40">
+    <action_set>
+      <rsc_op id="31" operation="monitor" operation_key="FAKECLONE1:3_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="41">
+    <action_set>
+      <rsc_op id="59" operation="monitor" operation_key="FAKECLONE1:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="58" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="42">
+    <action_set>
+      <rsc_op id="58" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="43">
+    <action_set>
+      <rsc_op id="61" operation="monitor" operation_key="FAKECLONE1:5_monitor_60000" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="44">
+    <action_set>
+      <rsc_op id="60" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="45">
+    <action_set>
+      <rsc_op id="11" operation="monitor" operation_key="FAKECLONE1:5_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="46" priority="1000000">
+    <action_set>
+      <pseudo_event id="63" operation="running" operation_key="FAKECLONE1-clone_running_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="50" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="52" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="54" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="56" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="58" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="47">
+    <action_set>
+      <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="48">
+    <action_set>
+      <rsc_op id="67" operation="monitor" operation_key="FAKECLONE2:0_monitor_60000" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="66" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="49">
+    <action_set>
+      <rsc_op id="66" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="50">
+    <action_set>
+      <rsc_op id="69" operation="monitor" operation_key="FAKECLONE2:1_monitor_60000" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="68" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="51">
+    <action_set>
+      <rsc_op id="68" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="52">
+    <action_set>
+      <rsc_op id="71" operation="monitor" operation_key="FAKECLONE2:2_monitor_60000" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="70" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="53">
+    <action_set>
+      <rsc_op id="70" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="54">
+    <action_set>
+      <rsc_op id="22" operation="monitor" operation_key="FAKECLONE2:2_monitor_0" on_node="18node3" on_node_uuid="3">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="55">
+    <action_set>
+      <rsc_op id="73" operation="monitor" operation_key="FAKECLONE2:3_monitor_60000" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="72" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="56">
+    <action_set>
+      <rsc_op id="72" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="57">
+    <action_set>
+      <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="58">
+    <action_set>
+      <rsc_op id="75" operation="monitor" operation_key="FAKECLONE2:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="74" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="59">
+    <action_set>
+      <rsc_op id="74" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:4" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="60">
+    <action_set>
+      <rsc_op id="77" operation="monitor" operation_key="FAKECLONE2:5_monitor_60000" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="61">
+    <action_set>
+      <rsc_op id="76" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="62">
+    <action_set>
+      <rsc_op id="12" operation="monitor" operation_key="FAKECLONE2:5_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="63" priority="1000000">
+    <action_set>
+      <pseudo_event id="79" operation="running" operation_key="FAKECLONE2-clone_running_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="66" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="68" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="70" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="72" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="74" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="64">
+    <action_set>
+      <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0">
+        <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="65" priority="1000000">
+    <action_set>
+      <rsc_op id="23" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="24" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="25" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="26" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="27" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="28" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="29" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="30" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="31" operation="monitor" operation_key="FAKECLONE1:3_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="66" priority="1000000">
+    <action_set>
+      <rsc_op id="13" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="14" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="15" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="16" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="17" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="18" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="19" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="20" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="21" operation="monitor" operation_key="FAKECLONE1:2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="22" operation="monitor" operation_key="FAKECLONE2:2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="67" priority="1000000">
+    <action_set>
+      <rsc_op id="8" operation="probe_complete" operation_key="probe_complete-18builder" on_node="18builder" on_node_uuid="5">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="9" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="10" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="11" operation="monitor" operation_key="FAKECLONE1:5_monitor_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="12" operation="monitor" operation_key="FAKECLONE2:5_monitor_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="68">
+    <action_set>
+      <pseudo_event id="7" operation="probe_nodes_complete" operation_key="probe_nodes_complete">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="8" operation="probe_complete" operation_key="probe_complete-18builder" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="13" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="23" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="69">
+    <action_set>
+      <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="70">
+    <action_set>
+      <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="39" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="42" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="45" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+</transition_graph>
diff --git a/pengine/test10/rsc-discovery-per-node.scores b/pengine/test10/rsc-discovery-per-node.scores
new file mode 100644
index 00000000000..a0ad11fa9aa
--- /dev/null
+++ b/pengine/test10/rsc-discovery-per-node.scores
@@ -0,0 +1,199 @@
+Allocation scores:
+clone_color: FAKECLONE1-clone allocation score on 18builder: 0
+clone_color: FAKECLONE1-clone allocation score on 18node1: 0
+clone_color: FAKECLONE1-clone allocation score on 18node2: 0
+clone_color: FAKECLONE1-clone allocation score on 18node3: 0
+clone_color: FAKECLONE1-clone allocation score on 18node4: 0
+clone_color: FAKECLONE1-clone allocation score on remote1: 0
+clone_color: FAKECLONE1:0 allocation score on 18builder: 0
+clone_color: FAKECLONE1:0 allocation score on 18node1: 0
+clone_color: FAKECLONE1:0 allocation score on 18node2: 0
+clone_color: FAKECLONE1:0 allocation score on 18node3: 0
+clone_color: FAKECLONE1:0 allocation score on 18node4: 0
+clone_color: FAKECLONE1:0 allocation score on remote1: 0
+clone_color: FAKECLONE1:1 allocation score on 18builder: 0
+clone_color: FAKECLONE1:1 allocation score on 18node1: 0
+clone_color: FAKECLONE1:1 allocation score on 18node2: 0
+clone_color: FAKECLONE1:1 allocation score on 18node3: 0
+clone_color: FAKECLONE1:1 allocation score on 18node4: 0
+clone_color: FAKECLONE1:1 allocation score on remote1: 0
+clone_color: FAKECLONE1:2 allocation score on 18builder: 0
+clone_color: FAKECLONE1:2 allocation score on 18node1: 0
+clone_color: FAKECLONE1:2 allocation score on 18node2: 0
+clone_color: FAKECLONE1:2 allocation score on 18node3: 0
+clone_color: FAKECLONE1:2 allocation score on 18node4: 0
+clone_color: FAKECLONE1:2 allocation score on remote1: 0
+clone_color: FAKECLONE1:3 allocation score on 18builder: 0
+clone_color: FAKECLONE1:3 allocation score on 18node1: 0
+clone_color: FAKECLONE1:3 allocation score on 18node2: 0
+clone_color: FAKECLONE1:3 allocation score on 18node3: 0
+clone_color: FAKECLONE1:3 allocation score on 18node4: 0
+clone_color: FAKECLONE1:3 allocation score on remote1: 0
+clone_color: FAKECLONE1:4 allocation score on 18builder: 0
+clone_color: FAKECLONE1:4 allocation score on 18node1: 0
+clone_color: FAKECLONE1:4 allocation score on 18node2: 0
+clone_color: FAKECLONE1:4 allocation score on 18node3: 0
+clone_color: FAKECLONE1:4 allocation score on 18node4: 0
+clone_color: FAKECLONE1:4 allocation score on remote1: 0
+clone_color: FAKECLONE1:5 allocation score on 18builder: 0
+clone_color: FAKECLONE1:5 allocation score on 18node1: 0
+clone_color: FAKECLONE1:5 allocation score on 18node2: 0
+clone_color: FAKECLONE1:5 allocation score on 18node3: 0
+clone_color: FAKECLONE1:5 allocation score on 18node4: 0
+clone_color: FAKECLONE1:5 allocation score on remote1: 0
+clone_color: FAKECLONE2-clone allocation score on 18builder: 0
+clone_color: FAKECLONE2-clone allocation score on 18node1: 0
+clone_color: FAKECLONE2-clone allocation score on 18node2: 0
+clone_color: FAKECLONE2-clone allocation score on 18node3: 0
+clone_color: FAKECLONE2-clone allocation score on 18node4: 0
+clone_color: FAKECLONE2-clone allocation score on remote1: 0
+clone_color: FAKECLONE2:0 allocation score on 18builder: 0
+clone_color: FAKECLONE2:0 allocation score on 18node1: 0
+clone_color: FAKECLONE2:0 allocation score on 18node2: 0
+clone_color: FAKECLONE2:0 allocation score on 18node3: 0
+clone_color: FAKECLONE2:0 allocation score on 18node4: 0
+clone_color: FAKECLONE2:0 allocation score on remote1: 0
+clone_color: FAKECLONE2:1 allocation score on 18builder: 0
+clone_color: FAKECLONE2:1 allocation score on 18node1: 0
+clone_color: FAKECLONE2:1 allocation score on 18node2: 0
+clone_color: FAKECLONE2:1 allocation score on 18node3: 0
+clone_color: FAKECLONE2:1 allocation score on 18node4: 0
+clone_color: FAKECLONE2:1 allocation score on remote1: 0
+clone_color: FAKECLONE2:2 allocation score on 18builder: 0
+clone_color: FAKECLONE2:2 allocation score on 18node1: 0
+clone_color: FAKECLONE2:2 allocation score on 18node2: 0
+clone_color: FAKECLONE2:2 allocation score on 18node3: 0
+clone_color: FAKECLONE2:2 allocation score on 18node4: 0
+clone_color: FAKECLONE2:2 allocation score on remote1: 0
+clone_color: FAKECLONE2:3 allocation score on 18builder: 0
+clone_color: FAKECLONE2:3 allocation score on 18node1: 0
+clone_color: FAKECLONE2:3 allocation score on 18node2: 0
+clone_color: FAKECLONE2:3 allocation score on 18node3: 0
+clone_color: FAKECLONE2:3 allocation score on 18node4: 0
+clone_color: FAKECLONE2:3 allocation score on remote1: 0
+clone_color: FAKECLONE2:4 allocation score on 18builder: 0
+clone_color: FAKECLONE2:4 allocation score on 18node1: 0
+clone_color: FAKECLONE2:4 allocation score on 18node2: 0
+clone_color: FAKECLONE2:4 allocation score on 18node3: 0
+clone_color: FAKECLONE2:4 allocation score on 18node4: 0
+clone_color: FAKECLONE2:4 allocation score on remote1: 0
+clone_color: FAKECLONE2:5 allocation score on 18builder: 0
+clone_color: FAKECLONE2:5 allocation score on 18node1: 0
+clone_color: FAKECLONE2:5 allocation score on 18node2: 0
+clone_color: FAKECLONE2:5 allocation score on 18node3: 0
+clone_color: FAKECLONE2:5 allocation score on 18node4: 0
+clone_color: FAKECLONE2:5 allocation score on remote1: 0
+native_color: FAKE1 allocation score on 18builder: 0
+native_color: FAKE1 allocation score on 18node1: 0
+native_color: FAKE1 allocation score on 18node2: 0
+native_color: FAKE1 allocation score on 18node3: 0
+native_color: FAKE1 allocation score on 18node4: 0
+native_color: FAKE1 allocation score on remote1: 0
+native_color: FAKE2 allocation score on 18builder: 0
+native_color: FAKE2 allocation score on 18node1: 0
+native_color: FAKE2 allocation score on 18node2: 0
+native_color: FAKE2 allocation score on 18node3: 0
+native_color: FAKE2 allocation score on 18node4: 0
+native_color: FAKE2 allocation score on remote1: 0
+native_color: FAKE3 allocation score on 18builder: 0
+native_color: FAKE3 allocation score on 18node1: 0
+native_color: FAKE3 allocation score on 18node2: 0
+native_color: FAKE3 allocation score on 18node3: 0
+native_color: FAKE3 allocation score on 18node4: 0
+native_color: FAKE3 allocation score on remote1: 0
+native_color: FAKE4 allocation score on 18builder: 0
+native_color: FAKE4 allocation score on 18node1: 0
+native_color: FAKE4 allocation score on 18node2: 0
+native_color: FAKE4 allocation score on 18node3: 0
+native_color: FAKE4 allocation score on 18node4: 0
+native_color: FAKE4 allocation score on remote1: 0
+native_color: FAKE5 allocation score on 18builder: 0
+native_color: FAKE5 allocation score on 18node1: 0
+native_color: FAKE5 allocation score on 18node2: 0
+native_color: FAKE5 allocation score on 18node3: 0
+native_color: FAKE5 allocation score on 18node4: 0
+native_color: FAKE5 allocation score on remote1: 0
+native_color: FAKECLONE1:0 allocation score on 18builder: 0
+native_color: FAKECLONE1:0 allocation score on 18node1: 0
+native_color: FAKECLONE1:0 allocation score on 18node2: 0
+native_color: FAKECLONE1:0 allocation score on 18node3: 0
+native_color: FAKECLONE1:0 allocation score on 18node4: 0
+native_color: FAKECLONE1:0 allocation score on remote1: 0
+native_color: FAKECLONE1:1 allocation score on 18builder: 0
+native_color: FAKECLONE1:1 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:1 allocation score on 18node2: 0
+native_color: FAKECLONE1:1 allocation score on 18node3: 0
+native_color: FAKECLONE1:1 allocation score on 18node4: 0
+native_color: FAKECLONE1:1 allocation score on remote1: 0
+native_color: FAKECLONE1:2 allocation score on 18builder: 0
+native_color: FAKECLONE1:2 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:2 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE1:2 allocation score on 18node3: 0
+native_color: FAKECLONE1:2 allocation score on 18node4: 0
+native_color: FAKECLONE1:2 allocation score on remote1: 0
+native_color: FAKECLONE1:3 allocation score on 18builder: 0
+native_color: FAKECLONE1:3 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:3 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE1:3 allocation score on 18node3: -INFINITY
+native_color: FAKECLONE1:3 allocation score on 18node4: 0
+native_color: FAKECLONE1:3 allocation score on remote1: 0
+native_color: FAKECLONE1:4 allocation score on 18builder: 0
+native_color: FAKECLONE1:4 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:4 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE1:4 allocation score on 18node3: -INFINITY
+native_color: FAKECLONE1:4 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE1:4 allocation score on remote1: 0
+native_color: FAKECLONE1:5 allocation score on 18builder: 0
+native_color: FAKECLONE1:5 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE1:5 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE1:5 allocation score on 18node3: -INFINITY
+native_color: FAKECLONE1:5 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE1:5 allocation score on remote1: -INFINITY
+native_color: FAKECLONE2:0 allocation score on 18builder: 0
+native_color: FAKECLONE2:0 allocation score on 18node1: 0
+native_color: FAKECLONE2:0 allocation score on 18node2: 0
+native_color: FAKECLONE2:0 allocation score on 18node3: 0
+native_color: FAKECLONE2:0 allocation score on 18node4: 0
+native_color: FAKECLONE2:0 allocation score on remote1: 0
+native_color: FAKECLONE2:1 allocation score on 18builder: 0
+native_color: FAKECLONE2:1 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:1 allocation score on 18node2: 0
+native_color: FAKECLONE2:1 allocation score on 18node3: 0
+native_color: FAKECLONE2:1 allocation score on 18node4: 0
+native_color: FAKECLONE2:1 allocation score on remote1: 0
+native_color: FAKECLONE2:2 allocation score on 18builder: 0
+native_color: FAKECLONE2:2 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:2 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE2:2 allocation score on 18node3: 0
+native_color: FAKECLONE2:2 allocation score on 18node4: 0
+native_color: FAKECLONE2:2 allocation score on remote1: 0
+native_color: FAKECLONE2:3 allocation score on 18builder: 0
+native_color: FAKECLONE2:3 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:3 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE2:3 allocation score on 18node3: -INFINITY
+native_color: FAKECLONE2:3 allocation score on 18node4: 0
+native_color: FAKECLONE2:3 allocation score on remote1: 0
+native_color: FAKECLONE2:4 allocation score on 18builder: 0
+native_color: FAKECLONE2:4 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:4 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE2:4 allocation score on 18node3: -INFINITY
+native_color: FAKECLONE2:4 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE2:4 allocation score on remote1: 0
+native_color: FAKECLONE2:5 allocation score on 18builder: 0
+native_color: FAKECLONE2:5 allocation score on 18node1: -INFINITY
+native_color: FAKECLONE2:5 allocation score on 18node2: -INFINITY
+native_color: FAKECLONE2:5 allocation score on 18node3: -INFINITY
+native_color: FAKECLONE2:5 allocation score on 18node4: -INFINITY
+native_color: FAKECLONE2:5 allocation score on remote1: -INFINITY
+native_color: remote1 allocation score on 18builder: 0
+native_color: remote1 allocation score on 18node1: 0
+native_color: remote1 allocation score on 18node2: 0
+native_color: remote1 allocation score on 18node3: 0
+native_color: remote1 allocation score on 18node4: 0
+native_color: remote1 allocation score on remote1: -INFINITY
+native_color: shooter allocation score on 18builder: 0
+native_color: shooter allocation score on 18node1: 0
+native_color: shooter allocation score on 18node2: 0
+native_color: shooter allocation score on 18node3: 0
+native_color: shooter allocation score on 18node4: 0
+native_color: shooter allocation score on remote1: -INFINITY
diff --git a/pengine/test10/rsc-discovery-per-node.summary b/pengine/test10/rsc-discovery-per-node.summary
new file mode 100644
index 00000000000..ede02c06d44
--- /dev/null
+++ b/pengine/test10/rsc-discovery-per-node.summary
@@ -0,0 +1,123 @@
+
+Current cluster status:
+Online: [ 18builder 18node1 18node2 18node3 18node4 ]
+RemoteOFFLINE: [ remote1 ]
+
+ shooter	(stonith:fence_xvm):	Started 18node1 
+ remote1	(ocf::pacemaker:remote):	Stopped 
+ FAKE1	(ocf::heartbeat:Dummy):	Stopped 
+ FAKE2	(ocf::heartbeat:Dummy):	Started 18node2 
+ FAKE3	(ocf::heartbeat:Dummy):	Started 18builder 
+ FAKE4	(ocf::heartbeat:Dummy):	Started 18node1 
+ FAKE5	(ocf::heartbeat:Dummy):	Stopped 
+ Clone Set: FAKECLONE1-clone [FAKECLONE1]
+     Stopped: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+     Stopped: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+
+Transition Summary:
+ * Start   remote1	(18builder)
+ * Start   FAKE1	(18node2)
+ * Move    FAKE2	(Started 18node2 -> 18node3)
+ * Move    FAKE3	(Started 18builder -> 18node4)
+ * Move    FAKE4	(Started 18node1 -> remote1)
+ * Start   FAKE5	(18builder)
+ * Start   FAKECLONE1:0	(18node1)
+ * Start   FAKECLONE1:1	(18node2)
+ * Start   FAKECLONE1:2	(18node3)
+ * Start   FAKECLONE1:3	(18node4)
+ * Start   FAKECLONE1:4	(remote1)
+ * Start   FAKECLONE1:5	(18builder)
+ * Start   FAKECLONE2:0	(18node1)
+ * Start   FAKECLONE2:1	(18node2)
+ * Start   FAKECLONE2:2	(18node3)
+ * Start   FAKECLONE2:3	(18node4)
+ * Start   FAKECLONE2:4	(remote1)
+ * Start   FAKECLONE2:5	(18builder)
+
+Executing cluster transition:
+ * Resource action: shooter         monitor on 18node4
+ * Resource action: shooter         monitor on 18node3
+ * Resource action: remote1         monitor on 18node4
+ * Resource action: remote1         monitor on 18node3
+ * Resource action: FAKE1           monitor on 18node4
+ * Resource action: FAKE1           monitor on 18node3
+ * Resource action: FAKE1           monitor on 18builder
+ * Resource action: FAKE2           monitor on 18node4
+ * Resource action: FAKE2           monitor on 18node3
+ * Resource action: FAKE3           monitor on 18node4
+ * Resource action: FAKE3           monitor on 18node3
+ * Resource action: FAKE4           monitor on 18node4
+ * Resource action: FAKE4           monitor on 18node3
+ * Resource action: FAKE5           monitor on 18node4
+ * Resource action: FAKE5           monitor on 18node3
+ * Resource action: FAKE5           monitor on 18builder
+ * Resource action: FAKECLONE1:2    monitor on 18node3
+ * Resource action: FAKECLONE1:3    monitor on 18node4
+ * Resource action: FAKECLONE1:5    monitor on 18builder
+ * Pseudo action:   FAKECLONE1-clone_start_0
+ * Resource action: FAKECLONE2:2    monitor on 18node3
+ * Resource action: FAKECLONE2:3    monitor on 18node4
+ * Resource action: FAKECLONE2:5    monitor on 18builder
+ * Pseudo action:   FAKECLONE2-clone_start_0
+ * Pseudo action:   probe_nodes_complete
+ * Pseudo action:   probe_complete
+ * Resource action: remote1         start on 18builder
+ * Resource action: FAKE1           start on 18node2
+ * Resource action: FAKE2           stop on 18node2
+ * Resource action: FAKE3           stop on 18builder
+ * Resource action: FAKE4           stop on 18node1
+ * Resource action: FAKE5           start on 18builder
+ * Resource action: FAKECLONE1:0    start on 18node1
+ * Resource action: FAKECLONE1:1    start on 18node2
+ * Resource action: FAKECLONE1:2    start on 18node3
+ * Resource action: FAKECLONE1:3    start on 18node4
+ * Resource action: FAKECLONE1:4    start on remote1
+ * Resource action: FAKECLONE1:5    start on 18builder
+ * Pseudo action:   FAKECLONE1-clone_running_0
+ * Resource action: FAKECLONE2:0    start on 18node1
+ * Resource action: FAKECLONE2:1    start on 18node2
+ * Resource action: FAKECLONE2:2    start on 18node3
+ * Resource action: FAKECLONE2:3    start on 18node4
+ * Resource action: FAKECLONE2:4    start on remote1
+ * Resource action: FAKECLONE2:5    start on 18builder
+ * Pseudo action:   FAKECLONE2-clone_running_0
+ * Pseudo action:   all_stopped
+ * Resource action: remote1         monitor=60000 on 18builder
+ * Resource action: FAKE1           monitor=60000 on 18node2
+ * Resource action: FAKE2           start on 18node3
+ * Resource action: FAKE3           start on 18node4
+ * Resource action: FAKE4           start on remote1
+ * Resource action: FAKE5           monitor=60000 on 18builder
+ * Resource action: FAKECLONE1:0    monitor=60000 on 18node1
+ * Resource action: FAKECLONE1:1    monitor=60000 on 18node2
+ * Resource action: FAKECLONE1:2    monitor=60000 on 18node3
+ * Resource action: FAKECLONE1:3    monitor=60000 on 18node4
+ * Resource action: FAKECLONE1:4    monitor=60000 on remote1
+ * Resource action: FAKECLONE1:5    monitor=60000 on 18builder
+ * Resource action: FAKECLONE2:0    monitor=60000 on 18node1
+ * Resource action: FAKECLONE2:1    monitor=60000 on 18node2
+ * Resource action: FAKECLONE2:2    monitor=60000 on 18node3
+ * Resource action: FAKECLONE2:3    monitor=60000 on 18node4
+ * Resource action: FAKECLONE2:4    monitor=60000 on remote1
+ * Resource action: FAKECLONE2:5    monitor=60000 on 18builder
+ * Resource action: FAKE2           monitor=60000 on 18node3
+ * Resource action: FAKE3           monitor=60000 on 18node4
+ * Resource action: FAKE4           monitor=60000 on remote1
+
+Revised cluster status:
+Online: [ 18builder 18node1 18node2 18node3 18node4 ]
+RemoteOnline: [ remote1 ]
+
+ shooter	(stonith:fence_xvm):	Started 18node1 
+ remote1	(ocf::pacemaker:remote):	Started 18builder 
+ FAKE1	(ocf::heartbeat:Dummy):	Started 18node2 
+ FAKE2	(ocf::heartbeat:Dummy):	Started 18node3 
+ FAKE3	(ocf::heartbeat:Dummy):	Started 18node4 
+ FAKE4	(ocf::heartbeat:Dummy):	Started remote1 
+ FAKE5	(ocf::heartbeat:Dummy):	Started 18builder 
+ Clone Set: FAKECLONE1-clone [FAKECLONE1]
+     Started: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+ Clone Set: FAKECLONE2-clone [FAKECLONE2]
+     Started: [ 18builder 18node1 18node2 18node3 18node4 remote1 ]
+
diff --git a/pengine/test10/rsc-discovery-per-node.xml b/pengine/test10/rsc-discovery-per-node.xml
new file mode 100644
index 00000000000..c111de81887
--- /dev/null
+++ b/pengine/test10/rsc-discovery-per-node.xml
@@ -0,0 +1,182 @@
+<cib epoch="11" num_updates="1" admin_epoch="0" validate-with="pacemaker-next" cib-last-written="Tue Sep  3 17:59:24 2013" update-origin="18builder" update-client="crm_resource" crm_feature_set="3.0.7" have-quorum="1" dc-uuid="5">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-ab50afe"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="18node1">
+        <instance_attributes id="instance_attributes-18node1">
+          <nvpair name="resource-discovery-enabled" id="discovery-18node1" value="false"/>
+        </instance_attributes>
+      </node>
+      <node id="2" uname="18node2"/>
+      <node id="3" uname="18node3"/>
+      <node id="4" uname="18node4"/>
+      <node id="5" uname="18builder"/>
+    </nodes>
+    <resources>
+      <primitive class="stonith" id="shooter" type="fence_xvm">
+        <instance_attributes id="shooter-instance_attributes"/>
+        <operations>
+          <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="remote1" provider="pacemaker" type="remote">
+        <instance_attributes id="remote1-instance_attributes"/>
+        <operations>
+          <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+        <meta_attributes id="remote1-meta_attributes"/>
+      </primitive>
+      <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE1-instance_attributes"/>
+        <meta_attributes id="FAKE1-meta_attributes">
+        </meta_attributes>
+        <operations>
+          <op id="FAKE1-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE2-instance_attributes"/>
+        <meta_attributes id="FAKE2-meta_attributes">
+        </meta_attributes>
+        <operations>
+          <op id="FAKE2-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE3-instance_attributes"/>
+        <operations>
+          <op id="FAKE3-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE4" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE4-instance_attributes"/>
+        <operations>
+          <op id="FAKE4-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="FAKE5" provider="heartbeat" type="Dummy">
+        <instance_attributes id="FAKE5-instance_attributes"/>
+        <operations>
+          <op id="FAKE5-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <clone id="FAKECLONE1-clone">
+        <primitive class="ocf" id="FAKECLONE1" provider="heartbeat" type="Dummy">
+          <instance_attributes id="FAKECLONE1-instance_attributes"/>
+          <operations>
+            <op id="FAKECLONE1-monitor-interval-60s" interval="60s" name="monitor"/>
+          </operations>
+        </primitive>
+      </clone>
+      <clone id="FAKECLONE2-clone">
+        <primitive class="ocf" id="FAKECLONE2" provider="heartbeat" type="Dummy">
+          <instance_attributes id="FAKECLONE2-instance_attributes"/>
+          <operations>
+            <op id="FAKECLONE2-monitor-interval-60s" interval="60s" name="monitor"/>
+          </operations>
+        </primitive>
+      </clone>
+    </resources>
+    <constraints>
+    </constraints>
+  </configuration>
+  <status>
+    <node_state id="5" uname="18builder" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <transient_attributes id="5">
+        <instance_attributes id="status-5">
+        </instance_attributes>
+      </transient_attributes>
+      <lrm id="5">
+        <lrm_resources>
+          <lrm_resource id="shooter" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="shooter_last_0" operation_key="shooter_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="11:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;11:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="16" rc-code="0" op-status="0" interval="0" last-run="1378248545" last-rc-change="1378248545" exec-time="1" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:3:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:3:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248534" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="remote1_last_0" operation_key="remote1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:6:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:6:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="13" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="remote1_monitor_60000" operation_key="remote1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:5:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:5:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="3" rc-code="0" op-status="0" interval="60000" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="7:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;7:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="39" rc-code="7" op-status="0" interval="0" last-run="1378249145" last-rc-change="1378249145" exec-time="61" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="18:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;18:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1378249147" last-rc-change="1378249147" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE3_monitor_60000" operation_key="FAKE3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="19:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;19:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="56" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249147" exec-time="10" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="61" rc-code="7" op-status="0" interval="0" last-run="1378249149" last-rc-change="1378249149" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="1" uname="18node1" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="shooter" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="shooter_last_0" operation_key="shooter_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;12:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1378248547" last-rc-change="1378248547" exec-time="19" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="shooter_monitor_60000" operation_key="shooter_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:4:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;13:4:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="12" rc-code="0" op-status="0" interval="60000" last-rc-change="1378248548" exec-time="8" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="9:9:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;9:9:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="25" rc-code="7" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE3_last_failure_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;10:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="30" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="32" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:11:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:11:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="33" rc-code="0" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="15" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="22:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;22:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="40" rc-code="0" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="36" queue-time="38" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE4_monitor_60000" operation_key="FAKE4_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="23:12:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;23:12:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="44" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249148" exec-time="14" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="1">
+        <instance_attributes id="status-1">
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="2" uname="18node2" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
+      <lrm id="2">
+        <lrm_resources>
+          <lrm_resource id="shooter" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="8:3:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;8:3:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1378248530" last-rc-change="1378248530" exec-time="980" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="remote1" type="remote" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="remote1_last_0" operation_key="remote1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="10:4:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;10:4:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="1" rc-code="7" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE2_last_0" operation_key="FAKE2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="16:9:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;16:9:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1378249144" last-rc-change="1378249144" exec-time="18" queue-time="9" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="FAKE2_monitor_60000" operation_key="FAKE2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="17:9:0:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:0;17:9:0:6277c962-3d23-450c-8410-560e51c1302b" call-id="22" rc-code="0" op-status="0" interval="60000" last-rc-change="1378249144" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE3" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE3_last_0" operation_key="FAKE3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="12:10:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;12:10:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="27" rc-code="7" op-status="0" interval="0" last-run="1378249146" last-rc-change="1378249146" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="FAKE4" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="FAKE4_last_0" operation_key="FAKE4_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.7" transition-key="13:12:7:6277c962-3d23-450c-8410-560e51c1302b" transition-magic="0:7;13:12:7:6277c962-3d23-450c-8410-560e51c1302b" call-id="32" rc-code="7" op-status="0" interval="0" last-run="1378249148" last-rc-change="1378249148" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="2">
+        <instance_attributes id="status-2">
+          <nvpair name="resource-discovery-enabled" id="discovery-18node2" value="false"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="3" uname="18node3" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member"/>
+    <node_state id="4" uname="18node4" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member"/>
+    <node_state remote_node="true" id="remote1" uname="remote1" crm-debug-origin="remote_init_cib_status">
+      <transient_attributes id="remote1">
+        <instance_attributes id="status-remote1">
+          <nvpair name="resource-discovery-enabled" id="discovery-18node2" value="false"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+  </status>
+</cib>

From b125a2e0566d97132244a05486b97eb7c8e356f5 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Wed, 8 Oct 2014 11:02:04 -0400
Subject: [PATCH 14/29] High: pengine: prevent disabling rsc discovery per node
 in certain situations

prevent disabling rsc discovery per node for cluster nodes
prevent disabling rsc discovery per node for baremetal nodes when
stonith is disabled.
---
 lib/pengine/unpack.c                          |  17 +-
 pengine/test10/rsc-discovery-per-node.dot     |  20 +
 pengine/test10/rsc-discovery-per-node.exp     | 518 +++++++++++-------
 pengine/test10/rsc-discovery-per-node.summary |   8 +
 4 files changed, 360 insertions(+), 203 deletions(-)

diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 395782ae2e0..9bfc88af751 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1031,8 +1031,8 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set)
 
             resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
             if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
-                crm_info("Node %s has resource discovery disabled", this_node->details->uname);
-                this_node->details->rsc_discovery_enabled = FALSE;
+                crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
+                    XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
             }
 
             crm_trace("determining node state");
@@ -1137,8 +1137,17 @@ unpack_remote_status(xmlNode * status, pe_working_set_t * data_set)
 
         resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
         if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
-            crm_info("Node %s has resource discovery disabled", this_node->details->uname);
-            this_node->details->rsc_discovery_enabled = FALSE;
+            if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
+                crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.",
+                    XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
+            } else {
+                /* if we're here, this is either a baremetal node and fencing is enabled,
+                 * or this is a container node which we don't care if fencing is enabled 
+                 * or not on. container nodes are 'fenced' by recovering the container resource
+                 * regardless of whether fencing is enabled. */
+                crm_info("Node %s has resource discovery disabled", this_node->details->uname);
+                this_node->details->rsc_discovery_enabled = FALSE;
+            }
         }
     }
 
diff --git a/pengine/test10/rsc-discovery-per-node.dot b/pengine/test10/rsc-discovery-per-node.dot
index 4ddaba10e04..8f5889a6e13 100644
--- a/pengine/test10/rsc-discovery-per-node.dot
+++ b/pengine/test10/rsc-discovery-per-node.dot
@@ -1,6 +1,10 @@
  digraph "g" {
 "FAKE1_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
 "FAKE1_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
+"FAKE1_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKE1_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
+"FAKE1_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
 "FAKE1_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
 "FAKE1_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
 "FAKE1_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
@@ -40,6 +44,10 @@
 "FAKE4_stop_0 18node1" [ style=bold color="green" fontcolor="black"]
 "FAKE5_monitor_0 18builder" -> "probe_complete 18builder" [ style = bold]
 "FAKE5_monitor_0 18builder" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
+"FAKE5_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKE5_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
+"FAKE5_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
 "FAKE5_monitor_0 18node3" -> "probe_complete 18node3" [ style = bold]
 "FAKE5_monitor_0 18node3" [ style=bold color="green" fontcolor="black"]
 "FAKE5_monitor_0 18node4" -> "probe_complete 18node4" [ style = bold]
@@ -56,10 +64,14 @@
 "FAKECLONE1-clone_start_0" -> "FAKECLONE1:4_start_0 remote1" [ style = bold]
 "FAKECLONE1-clone_start_0" -> "FAKECLONE1:5_start_0 18builder" [ style = bold]
 "FAKECLONE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE1:0_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
+"FAKECLONE1:0_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE1:0_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE1:0_start_0 18node1" -> "FAKECLONE1-clone_running_0" [ style = bold]
 "FAKECLONE1:0_start_0 18node1" -> "FAKECLONE1:0_monitor_60000 18node1" [ style = bold]
 "FAKECLONE1:0_start_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE1:1_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
+"FAKECLONE1:1_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE1:1_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE1:1_start_0 18node2" -> "FAKECLONE1-clone_running_0" [ style = bold]
 "FAKECLONE1:1_start_0 18node2" -> "FAKECLONE1:1_monitor_60000 18node2" [ style = bold]
@@ -95,10 +107,14 @@
 "FAKECLONE2-clone_start_0" -> "FAKECLONE2:4_start_0 remote1" [ style = bold]
 "FAKECLONE2-clone_start_0" -> "FAKECLONE2:5_start_0 18builder" [ style = bold]
 "FAKECLONE2-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"FAKECLONE2:0_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
+"FAKECLONE2:0_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE2:0_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE2:0_start_0 18node1" -> "FAKECLONE2-clone_running_0" [ style = bold]
 "FAKECLONE2:0_start_0 18node1" -> "FAKECLONE2:0_monitor_60000 18node1" [ style = bold]
 "FAKECLONE2:0_start_0 18node1" [ style=bold color="green" fontcolor="black"]
+"FAKECLONE2:1_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
+"FAKECLONE2:1_monitor_0 18node2" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE2:1_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"]
 "FAKECLONE2:1_start_0 18node2" -> "FAKECLONE2-clone_running_0" [ style = bold]
 "FAKECLONE2:1_start_0 18node2" -> "FAKECLONE2:1_monitor_60000 18node2" [ style = bold]
@@ -128,6 +144,10 @@
 "all_stopped" [ style=bold color="green" fontcolor="orange"]
 "probe_complete 18builder" -> "probe_nodes_complete" [ style = bold]
 "probe_complete 18builder" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node1" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node1" [ style=bold color="green" fontcolor="black"]
+"probe_complete 18node2" -> "probe_nodes_complete" [ style = bold]
+"probe_complete 18node2" [ style=bold color="green" fontcolor="black"]
 "probe_complete 18node3" -> "probe_nodes_complete" [ style = bold]
 "probe_complete 18node3" [ style=bold color="green" fontcolor="black"]
 "probe_complete 18node4" -> "probe_nodes_complete" [ style = bold]
diff --git a/pengine/test10/rsc-discovery-per-node.exp b/pengine/test10/rsc-discovery-per-node.exp
index 0ffd3a1c399..8de04cd4c70 100644
--- a/pengine/test10/rsc-discovery-per-node.exp
+++ b/pengine/test10/rsc-discovery-per-node.exp
@@ -1,7 +1,7 @@
 <transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
   <synapse id="0">
     <action_set>
-      <rsc_op id="24" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="34" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="shooter" class="stonith" type="fence_xvm"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -10,7 +10,7 @@
   </synapse>
   <synapse id="1">
     <action_set>
-      <rsc_op id="14" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="24" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="shooter" class="stonith" type="fence_xvm"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -19,20 +19,20 @@
   </synapse>
   <synapse id="2">
     <action_set>
-      <rsc_op id="36" operation="monitor" operation_key="remote1_monitor_60000" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="46" operation="monitor" operation_key="remote1_monitor_60000" on_node="18builder" on_node_uuid="5">
         <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
         <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
     </inputs>
   </synapse>
   <synapse id="3">
     <action_set>
-      <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5">
         <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -45,7 +45,7 @@
   </synapse>
   <synapse id="4">
     <action_set>
-      <rsc_op id="25" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="35" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -54,7 +54,7 @@
   </synapse>
   <synapse id="5">
     <action_set>
-      <rsc_op id="15" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="25" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="remote1" class="ocf" provider="pacemaker" type="remote"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -63,20 +63,20 @@
   </synapse>
   <synapse id="6">
     <action_set>
-      <rsc_op id="38" operation="monitor" operation_key="FAKE1_monitor_60000" on_node="18node2" on_node_uuid="2">
+      <rsc_op id="48" operation="monitor" operation_key="FAKE1_monitor_60000" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="37" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2"/>
+        <rsc_op id="47" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
     </inputs>
   </synapse>
   <synapse id="7">
     <action_set>
-      <rsc_op id="37" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2">
+      <rsc_op id="47" operation="start" operation_key="FAKE1_start_0" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -89,7 +89,7 @@
   </synapse>
   <synapse id="8">
     <action_set>
-      <rsc_op id="26" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="36" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -98,7 +98,7 @@
   </synapse>
   <synapse id="9">
     <action_set>
-      <rsc_op id="16" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="26" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -107,7 +107,7 @@
   </synapse>
   <synapse id="10">
     <action_set>
-      <rsc_op id="9" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="19" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -116,20 +116,38 @@
   </synapse>
   <synapse id="11">
     <action_set>
-      <rsc_op id="41" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="14" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="12">
+    <action_set>
+      <rsc_op id="9" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18builder" on_node_uuid="5">
+        <primitive id="FAKE1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="13">
+    <action_set>
+      <rsc_op id="51" operation="monitor" operation_key="FAKE2_monitor_60000" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="40" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="50" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="12">
+  <synapse id="14">
     <action_set>
-      <rsc_op id="40" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="50" operation="start" operation_key="FAKE2_start_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -139,13 +157,13 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <rsc_op id="39" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
+        <rsc_op id="49" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="13">
+  <synapse id="15">
     <action_set>
-      <rsc_op id="39" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2">
+      <rsc_op id="49" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -156,40 +174,40 @@
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="14">
+  <synapse id="16">
     <action_set>
-      <rsc_op id="27" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="37" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="15">
+  <synapse id="17">
     <action_set>
-      <rsc_op id="17" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="27" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKE2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="16">
+  <synapse id="18">
     <action_set>
-      <rsc_op id="44" operation="monitor" operation_key="FAKE3_monitor_60000" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="54" operation="monitor" operation_key="FAKE3_monitor_60000" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="43" operation="start" operation_key="FAKE3_start_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="53" operation="start" operation_key="FAKE3_start_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="17">
+  <synapse id="19">
     <action_set>
-      <rsc_op id="43" operation="start" operation_key="FAKE3_start_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="53" operation="start" operation_key="FAKE3_start_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -199,13 +217,13 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <rsc_op id="42" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="52" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="18">
+  <synapse id="20">
     <action_set>
-      <rsc_op id="42" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="52" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -216,43 +234,43 @@
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="19">
+  <synapse id="21">
     <action_set>
-      <rsc_op id="28" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="38" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="20">
+  <synapse id="22">
     <action_set>
-      <rsc_op id="18" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="28" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKE3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="21">
+  <synapse id="23">
     <action_set>
-      <rsc_op id="47" operation="monitor" operation_key="FAKE4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+      <rsc_op id="57" operation="monitor" operation_key="FAKE4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
         <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <rsc_op id="46" operation="start" operation_key="FAKE4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+        <rsc_op id="56" operation="start" operation_key="FAKE4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="22">
+  <synapse id="24">
     <action_set>
-      <rsc_op id="46" operation="start" operation_key="FAKE4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+      <rsc_op id="56" operation="start" operation_key="FAKE4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
         <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -262,16 +280,16 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <rsc_op id="45" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
+        <rsc_op id="55" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="23">
+  <synapse id="25">
     <action_set>
-      <rsc_op id="45" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1">
+      <rsc_op id="55" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1">
         <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -282,40 +300,40 @@
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="24">
+  <synapse id="26">
     <action_set>
-      <rsc_op id="29" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="39" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="25">
+  <synapse id="27">
     <action_set>
-      <rsc_op id="19" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="29" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKE4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="26">
+  <synapse id="28">
     <action_set>
-      <rsc_op id="49" operation="monitor" operation_key="FAKE5_monitor_60000" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="59" operation="monitor" operation_key="FAKE5_monitor_60000" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="48" operation="start" operation_key="FAKE5_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="58" operation="start" operation_key="FAKE5_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="27">
+  <synapse id="29">
     <action_set>
-      <rsc_op id="48" operation="start" operation_key="FAKE5_start_0" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="58" operation="start" operation_key="FAKE5_start_0" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_timeout="20000" />
       </rsc_op>
@@ -326,25 +344,43 @@
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="28">
+  <synapse id="30">
     <action_set>
-      <rsc_op id="30" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="40" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="29">
+  <synapse id="31">
     <action_set>
-      <rsc_op id="20" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="30" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="30">
+  <synapse id="32">
+    <action_set>
+      <rsc_op id="20" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="33">
+    <action_set>
+      <rsc_op id="15" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="34">
     <action_set>
       <rsc_op id="10" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKE5" class="ocf" provider="heartbeat" type="Dummy"/>
@@ -353,22 +389,22 @@
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="31">
+  <synapse id="35">
     <action_set>
-      <rsc_op id="51" operation="monitor" operation_key="FAKECLONE1:0_monitor_60000" on_node="18node1" on_node_uuid="1">
+      <rsc_op id="61" operation="monitor" operation_key="FAKECLONE1:0_monitor_60000" on_node="18node1" on_node_uuid="1">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="50" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1"/>
+        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="32">
+  <synapse id="36">
     <action_set>
-      <rsc_op id="50" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1">
+      <rsc_op id="60" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -378,26 +414,35 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+        <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="33">
+  <synapse id="37">
+    <action_set>
+      <rsc_op id="16" operation="monitor" operation_key="FAKECLONE1:0_monitor_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="38">
     <action_set>
-      <rsc_op id="53" operation="monitor" operation_key="FAKECLONE1:1_monitor_60000" on_node="18node2" on_node_uuid="2">
+      <rsc_op id="63" operation="monitor" operation_key="FAKECLONE1:1_monitor_60000" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="52" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2"/>
+        <rsc_op id="62" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="34">
+  <synapse id="39">
     <action_set>
-      <rsc_op id="52" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2">
+      <rsc_op id="62" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -407,26 +452,35 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+        <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="35">
+  <synapse id="40">
+    <action_set>
+      <rsc_op id="21" operation="monitor" operation_key="FAKECLONE1:1_monitor_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE1" long-id="FAKECLONE1:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="41">
     <action_set>
-      <rsc_op id="55" operation="monitor" operation_key="FAKECLONE1:2_monitor_60000" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="65" operation="monitor" operation_key="FAKECLONE1:2_monitor_60000" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="54" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="64" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="36">
+  <synapse id="42">
     <action_set>
-      <rsc_op id="54" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="64" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -436,35 +490,35 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+        <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="37">
+  <synapse id="43">
     <action_set>
-      <rsc_op id="21" operation="monitor" operation_key="FAKECLONE1:2_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="31" operation="monitor" operation_key="FAKECLONE1:2_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="38">
+  <synapse id="44">
     <action_set>
-      <rsc_op id="57" operation="monitor" operation_key="FAKECLONE1:3_monitor_60000" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="67" operation="monitor" operation_key="FAKECLONE1:3_monitor_60000" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="56" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="66" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="39">
+  <synapse id="45">
     <action_set>
-      <rsc_op id="56" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="66" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -474,38 +528,38 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+        <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="40">
+  <synapse id="46">
     <action_set>
-      <rsc_op id="31" operation="monitor" operation_key="FAKECLONE1:3_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="41" operation="monitor" operation_key="FAKECLONE1:3_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="41">
+  <synapse id="47">
     <action_set>
-      <rsc_op id="59" operation="monitor" operation_key="FAKECLONE1:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+      <rsc_op id="69" operation="monitor" operation_key="FAKECLONE1:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <rsc_op id="58" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+        <rsc_op id="68" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="42">
+  <synapse id="48">
     <action_set>
-      <rsc_op id="58" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+      <rsc_op id="68" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -515,29 +569,29 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+        <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="43">
+  <synapse id="49">
     <action_set>
-      <rsc_op id="61" operation="monitor" operation_key="FAKECLONE1:5_monitor_60000" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="71" operation="monitor" operation_key="FAKECLONE1:5_monitor_60000" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="70" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="44">
+  <synapse id="50">
     <action_set>
-      <rsc_op id="60" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="70" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -547,11 +601,11 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+        <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="45">
+  <synapse id="51">
     <action_set>
       <rsc_op id="11" operation="monitor" operation_key="FAKECLONE1:5_monitor_0" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKECLONE1" long-id="FAKECLONE1:5" class="ocf" provider="heartbeat" type="Dummy"/>
@@ -560,60 +614,60 @@
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="46" priority="1000000">
+  <synapse id="52" priority="1000000">
     <action_set>
-      <pseudo_event id="63" operation="running" operation_key="FAKECLONE1-clone_running_0">
+      <pseudo_event id="73" operation="running" operation_key="FAKECLONE1-clone_running_0">
         <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </pseudo_event>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="50" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1"/>
+        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:0_start_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
       <trigger>
-        <rsc_op id="52" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2"/>
+        <rsc_op id="62" operation="start" operation_key="FAKECLONE1:1_start_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
       <trigger>
-        <rsc_op id="54" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="64" operation="start" operation_key="FAKECLONE1:2_start_0" on_node="18node3" on_node_uuid="3"/>
       </trigger>
       <trigger>
-        <rsc_op id="56" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="66" operation="start" operation_key="FAKECLONE1:3_start_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="58" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+        <rsc_op id="68" operation="start" operation_key="FAKECLONE1:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
       </trigger>
       <trigger>
-        <rsc_op id="60" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="70" operation="start" operation_key="FAKECLONE1:5_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
+        <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="47">
+  <synapse id="53">
     <action_set>
-      <pseudo_event id="62" operation="start" operation_key="FAKECLONE1-clone_start_0">
+      <pseudo_event id="72" operation="start" operation_key="FAKECLONE1-clone_start_0">
         <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </pseudo_event>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="48">
+  <synapse id="54">
     <action_set>
-      <rsc_op id="67" operation="monitor" operation_key="FAKECLONE2:0_monitor_60000" on_node="18node1" on_node_uuid="1">
+      <rsc_op id="77" operation="monitor" operation_key="FAKECLONE2:0_monitor_60000" on_node="18node1" on_node_uuid="1">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="66" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1"/>
+        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="49">
+  <synapse id="55">
     <action_set>
-      <rsc_op id="66" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1">
+      <rsc_op id="76" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -623,26 +677,35 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+        <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="50">
+  <synapse id="56">
+    <action_set>
+      <rsc_op id="17" operation="monitor" operation_key="FAKECLONE2:0_monitor_0" on_node="18node1" on_node_uuid="1">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:0" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="57">
     <action_set>
-      <rsc_op id="69" operation="monitor" operation_key="FAKECLONE2:1_monitor_60000" on_node="18node2" on_node_uuid="2">
+      <rsc_op id="79" operation="monitor" operation_key="FAKECLONE2:1_monitor_60000" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="68" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2"/>
+        <rsc_op id="78" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="51">
+  <synapse id="58">
     <action_set>
-      <rsc_op id="68" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2">
+      <rsc_op id="78" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -652,26 +715,35 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+        <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="52">
+  <synapse id="59">
+    <action_set>
+      <rsc_op id="22" operation="monitor" operation_key="FAKECLONE2:1_monitor_0" on_node="18node2" on_node_uuid="2">
+        <primitive id="FAKECLONE2" long-id="FAKECLONE2:1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="60">
     <action_set>
-      <rsc_op id="71" operation="monitor" operation_key="FAKECLONE2:2_monitor_60000" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="81" operation="monitor" operation_key="FAKECLONE2:2_monitor_60000" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="70" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="80" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="53">
+  <synapse id="61">
     <action_set>
-      <rsc_op id="70" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="80" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -681,35 +753,35 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+        <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="54">
+  <synapse id="62">
     <action_set>
-      <rsc_op id="22" operation="monitor" operation_key="FAKECLONE2:2_monitor_0" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:2_monitor_0" on_node="18node3" on_node_uuid="3">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:2" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="2" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="55">
+  <synapse id="63">
     <action_set>
-      <rsc_op id="73" operation="monitor" operation_key="FAKECLONE2:3_monitor_60000" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="83" operation="monitor" operation_key="FAKECLONE2:3_monitor_60000" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="72" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="82" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="56">
+  <synapse id="64">
     <action_set>
-      <rsc_op id="72" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="82" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -719,38 +791,38 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+        <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="57">
+  <synapse id="65">
     <action_set>
-      <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="42" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:3" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="3" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="58">
+  <synapse id="66">
     <action_set>
-      <rsc_op id="75" operation="monitor" operation_key="FAKECLONE2:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+      <rsc_op id="85" operation="monitor" operation_key="FAKECLONE2:4_monitor_60000" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <rsc_op id="74" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+        <rsc_op id="84" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="59">
+  <synapse id="67">
     <action_set>
-      <rsc_op id="74" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
+      <rsc_op id="84" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:4" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="4" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -760,29 +832,29 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <rsc_op id="35" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="45" operation="start" operation_key="remote1_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+        <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="60">
+  <synapse id="68">
     <action_set>
-      <rsc_op id="77" operation="monitor" operation_key="FAKECLONE2:5_monitor_60000" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="87" operation="monitor" operation_key="FAKECLONE2:5_monitor_60000" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="86" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="61">
+  <synapse id="69">
     <action_set>
-      <rsc_op id="76" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5">
+      <rsc_op id="86" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
         <attributes CRM_meta_clone="5" CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </rsc_op>
@@ -792,11 +864,11 @@
         <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+        <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="62">
+  <synapse id="70">
     <action_set>
       <rsc_op id="12" operation="monitor" operation_key="FAKECLONE2:5_monitor_0" on_node="18builder" on_node_uuid="5">
         <primitive id="FAKECLONE2" long-id="FAKECLONE2:5" class="ocf" provider="heartbeat" type="Dummy"/>
@@ -805,117 +877,159 @@
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="63" priority="1000000">
+  <synapse id="71" priority="1000000">
     <action_set>
-      <pseudo_event id="79" operation="running" operation_key="FAKECLONE2-clone_running_0">
+      <pseudo_event id="89" operation="running" operation_key="FAKECLONE2-clone_running_0">
         <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </pseudo_event>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="66" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1"/>
+        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:0_start_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
       <trigger>
-        <rsc_op id="68" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2"/>
+        <rsc_op id="78" operation="start" operation_key="FAKECLONE2:1_start_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
       <trigger>
-        <rsc_op id="70" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="80" operation="start" operation_key="FAKECLONE2:2_start_0" on_node="18node3" on_node_uuid="3"/>
       </trigger>
       <trigger>
-        <rsc_op id="72" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="82" operation="start" operation_key="FAKECLONE2:3_start_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="74" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
+        <rsc_op id="84" operation="start" operation_key="FAKECLONE2:4_start_0" on_node="remote1" on_node_uuid="remote1" router_node="18builder"/>
       </trigger>
       <trigger>
-        <rsc_op id="76" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="86" operation="start" operation_key="FAKECLONE2:5_start_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
+        <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="64">
+  <synapse id="72">
     <action_set>
-      <pseudo_event id="78" operation="start" operation_key="FAKECLONE2-clone_start_0">
+      <pseudo_event id="88" operation="start" operation_key="FAKECLONE2-clone_start_0">
         <attributes CRM_meta_clone_max="6" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
       </pseudo_event>
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="65" priority="1000000">
+  <synapse id="73" priority="1000000">
     <action_set>
-      <rsc_op id="23" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4">
+      <rsc_op id="33" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4">
         <attributes CRM_meta_op_no_wait="true" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="24" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="34" operation="monitor" operation_key="shooter_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="25" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="35" operation="monitor" operation_key="remote1_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="26" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="36" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="27" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="37" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="28" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="38" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="29" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="39" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="30" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="40" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="31" operation="monitor" operation_key="FAKECLONE1:3_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="41" operation="monitor" operation_key="FAKECLONE1:3_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
       <trigger>
-        <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="42" operation="monitor" operation_key="FAKECLONE2:3_monitor_0" on_node="18node4" on_node_uuid="4"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="66" priority="1000000">
+  <synapse id="74" priority="1000000">
     <action_set>
-      <rsc_op id="13" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3">
+      <rsc_op id="23" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3">
         <attributes CRM_meta_op_no_wait="true" />
       </rsc_op>
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="14" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="24" operation="monitor" operation_key="shooter_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="25" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="26" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="27" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="28" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="29" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="30" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="31" operation="monitor" operation_key="FAKECLONE1:2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="32" operation="monitor" operation_key="FAKECLONE2:2_monitor_0" on_node="18node3" on_node_uuid="3"/>
       </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="75" priority="1000000">
+    <action_set>
+      <rsc_op id="18" operation="probe_complete" operation_key="probe_complete-18node2" on_node="18node2" on_node_uuid="2">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
       <trigger>
-        <rsc_op id="15" operation="monitor" operation_key="remote1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="19" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
       <trigger>
-        <rsc_op id="16" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="20" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
       <trigger>
-        <rsc_op id="17" operation="monitor" operation_key="FAKE2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="21" operation="monitor" operation_key="FAKECLONE1:1_monitor_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
       <trigger>
-        <rsc_op id="18" operation="monitor" operation_key="FAKE3_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="22" operation="monitor" operation_key="FAKECLONE2:1_monitor_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="76" priority="1000000">
+    <action_set>
+      <rsc_op id="13" operation="probe_complete" operation_key="probe_complete-18node1" on_node="18node1" on_node_uuid="1">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs>
       <trigger>
-        <rsc_op id="19" operation="monitor" operation_key="FAKE4_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="14" operation="monitor" operation_key="FAKE1_monitor_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
       <trigger>
-        <rsc_op id="20" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="15" operation="monitor" operation_key="FAKE5_monitor_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
       <trigger>
-        <rsc_op id="21" operation="monitor" operation_key="FAKECLONE1:2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="16" operation="monitor" operation_key="FAKECLONE1:0_monitor_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
       <trigger>
-        <rsc_op id="22" operation="monitor" operation_key="FAKECLONE2:2_monitor_0" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="17" operation="monitor" operation_key="FAKECLONE2:0_monitor_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="67" priority="1000000">
+  <synapse id="77" priority="1000000">
     <action_set>
       <rsc_op id="8" operation="probe_complete" operation_key="probe_complete-18builder" on_node="18builder" on_node_uuid="5">
         <attributes CRM_meta_op_no_wait="true" />
@@ -936,7 +1050,7 @@
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="68">
+  <synapse id="78">
     <action_set>
       <pseudo_event id="7" operation="probe_nodes_complete" operation_key="probe_nodes_complete">
         <attributes />
@@ -947,14 +1061,20 @@
         <rsc_op id="8" operation="probe_complete" operation_key="probe_complete-18builder" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <rsc_op id="13" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3"/>
+        <rsc_op id="13" operation="probe_complete" operation_key="probe_complete-18node1" on_node="18node1" on_node_uuid="1"/>
       </trigger>
       <trigger>
-        <rsc_op id="23" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4"/>
+        <rsc_op id="18" operation="probe_complete" operation_key="probe_complete-18node2" on_node="18node2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="23" operation="probe_complete" operation_key="probe_complete-18node3" on_node="18node3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="33" operation="probe_complete" operation_key="probe_complete-18node4" on_node="18node4" on_node_uuid="4"/>
       </trigger>
     </inputs>
   </synapse>
-  <synapse id="69">
+  <synapse id="79">
     <action_set>
       <pseudo_event id="6" operation="probe_complete" operation_key="probe_complete">
         <attributes />
@@ -962,7 +1082,7 @@
     </action_set>
     <inputs/>
   </synapse>
-  <synapse id="70">
+  <synapse id="80">
     <action_set>
       <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped">
         <attributes />
@@ -970,13 +1090,13 @@
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="39" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
+        <rsc_op id="49" operation="stop" operation_key="FAKE2_stop_0" on_node="18node2" on_node_uuid="2"/>
       </trigger>
       <trigger>
-        <rsc_op id="42" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
+        <rsc_op id="52" operation="stop" operation_key="FAKE3_stop_0" on_node="18builder" on_node_uuid="5"/>
       </trigger>
       <trigger>
-        <rsc_op id="45" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
+        <rsc_op id="55" operation="stop" operation_key="FAKE4_stop_0" on_node="18node1" on_node_uuid="1"/>
       </trigger>
     </inputs>
   </synapse>
diff --git a/pengine/test10/rsc-discovery-per-node.summary b/pengine/test10/rsc-discovery-per-node.summary
index ede02c06d44..69d5ee3352e 100644
--- a/pengine/test10/rsc-discovery-per-node.summary
+++ b/pengine/test10/rsc-discovery-per-node.summary
@@ -42,6 +42,8 @@ Executing cluster transition:
  * Resource action: remote1         monitor on 18node3
  * Resource action: FAKE1           monitor on 18node4
  * Resource action: FAKE1           monitor on 18node3
+ * Resource action: FAKE1           monitor on 18node2
+ * Resource action: FAKE1           monitor on 18node1
  * Resource action: FAKE1           monitor on 18builder
  * Resource action: FAKE2           monitor on 18node4
  * Resource action: FAKE2           monitor on 18node3
@@ -51,11 +53,17 @@ Executing cluster transition:
  * Resource action: FAKE4           monitor on 18node3
  * Resource action: FAKE5           monitor on 18node4
  * Resource action: FAKE5           monitor on 18node3
+ * Resource action: FAKE5           monitor on 18node2
+ * Resource action: FAKE5           monitor on 18node1
  * Resource action: FAKE5           monitor on 18builder
+ * Resource action: FAKECLONE1:0    monitor on 18node1
+ * Resource action: FAKECLONE1:1    monitor on 18node2
  * Resource action: FAKECLONE1:2    monitor on 18node3
  * Resource action: FAKECLONE1:3    monitor on 18node4
  * Resource action: FAKECLONE1:5    monitor on 18builder
  * Pseudo action:   FAKECLONE1-clone_start_0
+ * Resource action: FAKECLONE2:0    monitor on 18node1
+ * Resource action: FAKECLONE2:1    monitor on 18node2
  * Resource action: FAKECLONE2:2    monitor on 18node3
  * Resource action: FAKECLONE2:3    monitor on 18node4
  * Resource action: FAKECLONE2:5    monitor on 18builder

From ada76264fe2cb243bfb90ea30c869268e144b194 Mon Sep 17 00:00:00 2001
From: "Gao,Yan" <ygao@suse.com>
Date: Wed, 8 Oct 2014 20:26:48 +0200
Subject: [PATCH 15/29] Fix: xml: Prevent assert errors in crm_element_value()
 on applying a patch without version information

If the patch is generated with "crm_diff --no-version/-u".
---
 lib/common/xml.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/lib/common/xml.c b/lib/common/xml.c
index 06de44c8407..51d6ada8450 100644
--- a/lib/common/xml.c
+++ b/lib/common/xml.c
@@ -1950,9 +1950,11 @@ bool xml_patch_versions(xmlNode *patchset, int add[3], int del[3])
             return -EINVAL;
     }
 
-    for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
-        crm_element_value_int(tmp, vfields[lpc], &(del[lpc]));
-        crm_trace("Got %d for del[%s]", del[lpc], vfields[lpc]);
+    if (tmp) {
+        for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
+            crm_element_value_int(tmp, vfields[lpc], &(del[lpc]));
+            crm_trace("Got %d for del[%s]", del[lpc], vfields[lpc]);
+        }
     }
 
     switch(format) {
@@ -1973,9 +1975,11 @@ bool xml_patch_versions(xmlNode *patchset, int add[3], int del[3])
             return -EINVAL;
     }
 
-    for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
-        crm_element_value_int(tmp, vfields[lpc], &(add[lpc]));
-        crm_trace("Got %d for add[%s]", add[lpc], vfields[lpc]);
+    if (tmp) {
+        for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
+            crm_element_value_int(tmp, vfields[lpc], &(add[lpc]));
+            crm_trace("Got %d for add[%s]", add[lpc], vfields[lpc]);
+        }
     }
 
     return pcmk_ok;

From 7cfe712adfd21f400b4944c8e27077983cca28ee Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Thu, 9 Oct 2014 16:50:43 -0400
Subject: [PATCH 16/29] Fix: pengine: allow remote-nodes to be placed in
 maintenance mode

---
 lib/pengine/unpack.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 9bfc88af751..df8ca691081 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1135,6 +1135,11 @@ unpack_remote_status(xmlNode * status, pe_working_set_t * data_set)
             this_node->details->standby = TRUE;
         }
 
+        if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance"))) {
+            crm_info("Node %s is in maintenance-mode", this_node->details->uname);
+            this_node->details->maintenance = TRUE;
+        }
+
         resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
         if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
             if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {

From 50ceea53bc1fa043c15a2cfa6c27fe10df830cc0 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Thu, 9 Oct 2014 17:30:45 -0400
Subject: [PATCH 17/29] Low: cts: move custom RemoteBaremetal pattern into
 patterns.py

---
 cts/CTStests.py | 2 +-
 cts/patterns.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/cts/CTStests.py b/cts/CTStests.py
index cd5b7ce365b..00d77e8d9ad 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -2753,7 +2753,7 @@ def step2_add_rsc(self, node):
         pats = [ ]
         watch = self.create_watch(pats, 120)
         watch.setwatch()
-        pats.append("process_lrm_event:.*Operation %s_start_0.*node=%s, .*confirmed.*true" % (self.remote_rsc, self.remote_node))
+        pats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "start", self.remote_node))
 
         # Add a resource that must live on remote-node
         self.add_primitive_rsc(node)
diff --git a/cts/patterns.py b/cts/patterns.py
index 8d34e1c0516..be3f4cd9e61 100644
--- a/cts/patterns.py
+++ b/cts/patterns.py
@@ -56,6 +56,7 @@ def __init__(self, name):
             "Pat:Fencing_ok"    : "stonith.*remote_op_done:.*Operation .* of %s by .*: OK",
 
             "Pat:RscOpOK"       : "process_lrm_event:.*Operation %s_%s.*ok.*confirmed",
+            "Pat:RscRemoteOpOK" : "process_lrm_event:.*Operation %s_%s.*ok.*node=%s, .*confirmed.*true",
         }
 
     def get_component(self, key):

From aeb21eda622496452f8c1ef9b1442b8fca545325 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Mon, 13 Oct 2014 14:23:31 -0400
Subject: [PATCH 18/29] Low: cts: Make the remotebaremetal test reusable as the
 base for other tests

---
 cts/CTStests.py | 39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/cts/CTStests.py b/cts/CTStests.py
index 00d77e8d9ad..6926682ded7 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -2634,11 +2634,11 @@ def errorstoignore(self):
 
 
 ###################################################################
-class RemoteBaremetal(CTSTest):
+class RemoteDriver(CTSTest):
 ###################################################################
     def __init__(self, cm):
         CTSTest.__init__(self,cm)
-        self.name = "RemoteBaremetal"
+        self.name = "RemoteDriver"
         self.is_docker_unsafe = 1
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
@@ -2907,6 +2907,39 @@ def errorstoignore(self):
                  """Failed to send remote""",
                 ]
 
-AllTestClasses.append(RemoteBaremetal)
+# Remote driver is called by other tests.
+
+###################################################################
+class RemoteBasic(CTSTest):
+###################################################################
+    def __init__(self, cm):
+        CTSTest.__init__(self,cm)
+        self.name = "RemoteBasic"
+        self.start = StartTest(cm)
+        self.startall = SimulStartLite(cm)
+        self.driver = RemoteDriver(cm)
+
+    def __call__(self, node):
+        '''Perform the 'RemoteBaremetal' test. '''
+        self.incr("calls")
+
+        ret = self.startall(None)
+        if not ret:
+            return self.failure("Setup failed, start all nodes failed.")
+
+        self.driver.setup_env()
+        self.driver.step1_start_metal(node)
+        self.driver.step2_add_rsc(node)
+        self.driver.step3_test_attributes(node)
+        self.driver.cleanup_metal(node)
+
+        self.debug("Waiting for the cluster to recover")
+        self.CM.cluster_stable()
+        if self.driver.failed == 1:
+            return self.failure(self.driver.fail_string)
+
+        return self.success()
+
+AllTestClasses.append(RemoteBasic)
 
 # vim:ts=4:sw=4:et:

From c38ef7c64eebd42458c68171f2f7770d6e0fc77a Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Mon, 13 Oct 2014 14:30:24 -0400
Subject: [PATCH 19/29] Low: cts: refactor fence watch patterns for pattern
 reuse

---
 cts/CTStests.py | 4 ++--
 cts/patterns.py | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cts/CTStests.py b/cts/CTStests.py
index 6926682ded7..e0c19406928 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -453,8 +453,8 @@ def __call__(self, node):
         is_dc = self.CM.is_node_dc(node)
 
         watchpats = []
-        watchpats.append("Operation .* for host '%s' with device .* returned: 0" % node)
-        watchpats.append("tengine_stonith_notify:.*Peer %s was terminated .*: OK" % node)
+        watchpats.append(self.templates["Pat:FenceOpOK"] % node)
+        watchpats.append(self.templates["Pat:NodeFenced"] % node)
 
         if self.Env["at-boot"] == 0:
             self.debug("Expecting %s to stay down" % node)
diff --git a/cts/patterns.py b/cts/patterns.py
index be3f4cd9e61..e734f40e0b1 100644
--- a/cts/patterns.py
+++ b/cts/patterns.py
@@ -57,6 +57,8 @@ def __init__(self, name):
 
             "Pat:RscOpOK"       : "process_lrm_event:.*Operation %s_%s.*ok.*confirmed",
             "Pat:RscRemoteOpOK" : "process_lrm_event:.*Operation %s_%s.*ok.*node=%s, .*confirmed.*true",
+            "Pat:NodeFenced"    : "tengine_stonith_notify:.*Peer %s was terminated .*: OK",
+            "Pat:FenceOpOK"     : "Operation .* for host '%s' with device .* returned: 0",
         }
 
     def get_component(self, key):

From 5f06256788f5c763b80d75762ba92cac2cf67265 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Tue, 14 Oct 2014 17:16:24 -0400
Subject: [PATCH 20/29] Fix: remote: report timestamps for remote connection
 resource operations

---
 crmd/remote_lrmd_ra.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
index f3dedeb3593..2f658eece56 100644
--- a/crmd/remote_lrmd_ra.c
+++ b/crmd/remote_lrmd_ra.c
@@ -140,8 +140,6 @@ recurring_helper(gpointer data)
 
         ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);
 
-        cmd->call_id = generate_callid();
-
         ra_data->cmds = g_list_append(ra_data->cmds, cmd);
         mainloop_set_trigger(ra_data->work);
     }
@@ -177,6 +175,24 @@ report_remote_ra_result(remote_ra_cmd_t * cmd)
     op.interval = cmd->interval;
     op.rc = cmd->rc;
     op.op_status = cmd->op_status;
+    op.t_run = cmd->start_time;
+    op.t_rcchange = cmd->start_time;
+    if (cmd->reported_success && cmd->rc != PCMK_OCF_OK) {
+        op.t_rcchange = time(NULL);
+        /* This edge case will likely never ever occur, but if it does the
+         * result is that a failure will not be processed correctly. This is only
+         * remotely possible because we are able to detect a connection resource's tcp
+         * connection has failed at any moment after start has completed. The actual
+         * recurring operation is just a connectivity ping.
+         *
+         * basically, we are not guaranteed that the first successful monitor op and
+         * a subsequent failed monitor op will not occur in the same timestamp. We have to
+         * make it look like the operations occurred at separate times though. */
+        if (op.t_rcchange == op.t_run) {
+            op.t_rcchange++;
+        }
+    }
+
     if (cmd->params) {
         lrmd_key_value_t *tmp;
 

From 2cbfd4a6729ce1e7eaa2b5f899fb227877997765 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Wed, 15 Oct 2014 15:07:02 -0400
Subject: [PATCH 21/29] Low: remote: advertise default port correctly, further
 adjust default timeout periods

---
 extra/resources/remote | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/extra/resources/remote b/extra/resources/remote
index 9f141a22897..d79c4c39341 100644
--- a/extra/resources/remote
+++ b/extra/resources/remote
@@ -58,12 +58,12 @@ meta_data() {
        tcp port to connect to.
     </longdesc>
     <shortdesc lang="en">tcp port</shortdesc>
-    <content type="string" default="1984"/>
+    <content type="string" default="3121"/>
     </parameter>
   </parameters>
   <actions>
-    <action name="start"   timeout="40" />
-    <action name="stop"    timeout="40" />
+    <action name="start"   timeout="60" />
+    <action name="stop"    timeout="60" />
     <action name="monitor"    timeout="30" />
     <action name="migrate_to"   timeout="60" />
     <action name="migrate_from" timeout="60" />

From 6cb1cbab7a85e51b1adabfbbf984e4ce20521c3c Mon Sep 17 00:00:00 2001
From: Hideo Yamauchi <renayama19661014@ybb.ne.jp>
Date: Thu, 16 Oct 2014 14:51:07 +0900
Subject: [PATCH 22/29] Change of the log level.

---
 fencing/remote.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fencing/remote.c b/fencing/remote.c
index 834c440970a..acfe571650f 100644
--- a/fencing/remote.c
+++ b/fencing/remote.c
@@ -277,7 +277,7 @@ remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup)
     }
 
     if (!op->delegate && data) {
-        xmlNode *ndata = get_xpath_object("//@" F_STONITH_DELEGATE, data, LOG_WARNING);
+        xmlNode *ndata = get_xpath_object("//@" F_STONITH_DELEGATE, data, LOG_TRACE);
         if(ndata) {
             op->delegate = crm_element_value_copy(ndata, F_STONITH_DELEGATE);
         }

From 61051d17f09a86d4e77241d3b244e770c5eef024 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Wed, 15 Oct 2014 16:58:12 -0400
Subject: [PATCH 23/29] Low: cts: baremetal remote fencing test

---
 cts/CIB.py      |   2 +
 cts/CTStests.py | 214 +++++++++++++++++++++++++++++++++++-------------
 2 files changed, 157 insertions(+), 59 deletions(-)

diff --git a/cts/CIB.py b/cts/CIB.py
index 6ce38e9ba32..d26efdbbe82 100644
--- a/cts/CIB.py
+++ b/cts/CIB.py
@@ -177,6 +177,8 @@ def contents(self, target=None):
                 for node in self.CM.Env["nodes"]:
                     ftype = self.CM.Env.RandomGen.choice(["levels-and", "levels-or ", "broadcast "])
                     self.CM.log(" - Using %s fencing for node: %s" % (ftype, node))
+                    # for baremetal remote node tests
+                    stt_nodes.append("remote_%s" % node)
                     if ftype == "levels-and":
                         stl.level(1, node, "FencingPass,Fencing")
                         stt_nodes.append(node)
diff --git a/cts/CTStests.py b/cts/CTStests.py
index e0c19406928..ae86e2ae390 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -2647,9 +2647,8 @@ def __init__(self, cm):
         self.failed = 0
         self.fail_string = ""
         self.remote_node_added = 0
-        self.remote_node = "remote1"
         self.remote_rsc_added = 0
-        self.remote_rsc = "remote1-rsc"
+        self.remote_rsc = "remote-rsc"
         self.cib_cmd = """cibadmin -C -o %s -X '%s' """
 
     def del_rsc(self, node, rsc):
@@ -2681,9 +2680,12 @@ def add_primitive_rsc(self, node):
         rsc_xml = """
 <primitive class="ocf" id="%s" provider="pacemaker" type="Dummy">
     <operations>
-      <op id="remote1-rsc-monitor-interval-10s" interval="10s" name="monitor"/>
+      <op id="remote-rsc-monitor-interval-10s" interval="10s" name="monitor"/>
     </operations>
-</primitive>""" % (self.remote_rsc)
+    <meta_attributes id="remote-meta_attributes">
+          <nvpair id="remote-meta-container" name="container" value="%s"/>
+    </meta_attributes id="remote-meta_attributes">
+</primitive>""" % (self.remote_rsc, self.remote_node)
         self.add_rsc(node, rsc_xml)
         if self.failed == 0:
             self.remote_rsc_added = 1
@@ -2691,21 +2693,38 @@ def add_primitive_rsc(self, node):
     def add_connection_rsc(self, node):
         rsc_xml = """
 <primitive class="ocf" id="%s" provider="pacemaker" type="remote">
-    <instance_attributes id="remote1-instance_attributes"/>
-        <instance_attributes id="remote1-instance_attributes">
-          <nvpair id="remote1-instance_attributes-server" name="server" value="%s"/>
+    <instance_attributes id="remote-instance_attributes"/>
+        <instance_attributes id="remote-instance_attributes">
+          <nvpair id="remote-instance_attributes-server" name="server" value="%s"/>
         </instance_attributes>
     <operations>
-      <op id="remote1-monitor-interval-60s" interval="60s" name="monitor"/>
-          <op id="remote1-name-start-interval-0-timeout-60" interval="0" name="start" timeout="60"/>
+      <op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
+      <op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="120"/>
     </operations>
-    <meta_attributes id="remote1-meta_attributes"/>
 </primitive>""" % (self.remote_node, node)
         self.add_rsc(node, rsc_xml)
         if self.failed == 0:
             self.remote_node_added = 1
 
-    def step1_start_metal(self, node):
+    def stop_pcmk_remote(self, node):
+        # disable pcmk remote
+        for i in range(10):
+            rc = self.rsh(node, "service pacemaker_remote stop")
+            if rc != 0:
+                time.sleep(6)
+            else:
+                break
+
+    def start_pcmk_remote(self, node):
+        for i in range(10):
+            rc = self.rsh(node, "service pacemaker_remote start")
+            if rc != 0:
+                time.sleep(6)
+            else:
+                self.pcmk_started = 1
+                break
+
+    def start_metal(self, node):
         pcmk_started = 0
 
         # make sure the resource doesn't already exist for some reason
@@ -2717,13 +2736,7 @@ def step1_start_metal(self, node):
             self.fail_string = "Failed to shutdown cluster node %s" % (node)
             return
 
-        for i in range(10):
-            rc = self.rsh(node, "service pacemaker_remote start")
-            if rc != 0:
-                time.sleep(6)
-            else:
-                self.pcmk_started = 1
-                break
+        self.start_pcmk_remote(node)
 
         if self.pcmk_started == 0:
             self.failed = 1
@@ -2735,6 +2748,7 @@ def step1_start_metal(self, node):
         watch = self.create_watch(pats, 120)
         watch.setwatch()
         pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "start"))
+        pats.append(self.templates["Pat:DC_IDLE"])
 
         self.add_connection_rsc(node)
 
@@ -2745,40 +2759,72 @@ def step1_start_metal(self, node):
             self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
             self.failed = 1
 
-    def step2_add_rsc(self, node):
+    def fail_connection(self, node):
         if self.failed == 1:
             return
 
-        # verify we can put a resource on the remote node
+        watchpats = [ ]
+        watchpats.append(self.templates["Pat:FenceOpOK"] % self.remote_node)
+        watchpats.append(self.templates["Pat:NodeFenced"] % self.remote_node)
+
+        watch = self.create_watch(watchpats, 120)
+        watch.setwatch()
+
+        # force stop the pcmk remote daemon. this will result in fencing
+        self.debug("Force stopped active remote node")
+        self.stop_pcmk_remote(node)
+
+        self.debug("Waiting for remote node to be fenced.")
+        self.set_timer("remoteMetalFence")
+        watch.lookforall()
+        self.log_timer("remoteMetalFence")
+        if watch.unmatched:
+            self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
+            self.logger.log(self.fail_string)
+            self.failed = 1
+            return
+
+        self.debug("Waiting for the remote node to come back up")
+        self.CM.ns.WaitForNodeToComeUp(node, 120);
+
         pats = [ ]
         watch = self.create_watch(pats, 120)
         watch.setwatch()
-        pats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "start", self.remote_node))
+        pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "start"))
+        if self.remote_rsc_added == 1:
+            pats.append(self.templates["Pat:RscOpOK"] % (self.remote_rsc, "monitor"))
 
-        # Add a resource that must live on remote-node
-        self.add_primitive_rsc(node)
-        # this crm_resource command actually occurs on the remote node
-        # which verifies that the ipc proxy works
-        time.sleep(1)
+        # start the remote node again watch it integrate back into cluster.
+        self.start_pcmk_remote(node)
+        if self.pcmk_started == 0:
+            self.failed = 1
+            self.fail_string = "Failed to start pacemaker_remote on node %s" % (node)
+            self.logger.log(self.fail_string)
+            return
 
-        (rc, lines) = self.rsh(node, "crm_resource -W -r remote1-rsc --quiet", None)
-        if rc != 0:
-            self.fail_string = "Failed to get location of resource remote1-rsc"
+        self.debug("Waiting for remote node to rejoin cluster after being fenced.")
+        self.set_timer("remoteMetalRestart")
+        watch.lookforall()
+        self.log_timer("remoteMetalRestart")
+        if watch.unmatched:
+            self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
             self.failed = 1
+            self.logger.log(self.fail_string)
             return
 
-        find = 0
-        for line in lines:
-            if self.remote_node in line.split():
-                find = 1
-                break
+    def add_dummy_rsc(self, node):
+        if self.failed == 1:
+            return
 
-        if find == 0:
-            rc = self.rsh(node, "crm_resource -M -r remote1-rsc -N %s" % (self.remote_node))
-            if rc != 0:
-                self.fail_string = "Failed to place primitive on remote-node"
-                self.failed = 1
-                return
+        # verify we can put a resource on the remote node
+        pats = [ ]
+        watch = self.create_watch(pats, 120)
+        watch.setwatch()
+        pats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "start", self.remote_node))
+        pats.append(self.templates["Pat:DC_IDLE"])
+
+        # Add a resource that must live on remote-node
+        self.add_primitive_rsc(node)
 
         self.set_timer("remoteMetalRsc")
         watch.lookforall()
@@ -2787,7 +2833,7 @@ def step2_add_rsc(self, node):
             self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
             self.failed = 1
 
-    def step3_test_attributes(self, node):
+    def test_attributes(self, node):
         if self.failed == 1:
             return
 
@@ -2827,7 +2873,7 @@ def cleanup_metal(self, node):
 
         self.set_timer("remoteMetalCleanup")
         if self.remote_rsc_added == 1:
-            self.rsh(node, "crm_resource -U -r remote1-rsc -N %s" % (self.remote_node))
+            self.rsh(node, "crm_resource -U -r %s -N %s" % (self.remote_rsc, self.remote_node))
             self.del_rsc(node, self.remote_rsc)
         if self.remote_node_added == 1:
             self.del_rsc(node, self.remote_node)
@@ -2838,15 +2884,11 @@ def cleanup_metal(self, node):
             self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
             self.failed = 1
 
-        # disable pcmk remote
-        for i in range(10):
-            rc = self.rsh(node, "service pacemaker_remote stop")
-            if rc != 0:
-                time.sleep(6)
-            else:
-                break
+        self.stop_pcmk_remote(node)
+
+    def setup_env(self, node):
 
-    def setup_env(self):
+        self.remote_node = "remote_%s" % (node)
         sync_key = 0
 
         # we are assuming if all nodes have a key, that it is
@@ -2887,10 +2929,10 @@ def __call__(self, node):
         if not ret:
             return self.failure("Setup failed, start all nodes failed.")
 
-        self.setup_env()
-        self.step1_start_metal(node)
-        self.step2_add_rsc(node)
-        self.step3_test_attributes(node)
+        self.setup_env(node)
+        self.start_metal(node)
+        self.add_dummy_rsc(node)
+        self.test_attributes(node)
         self.cleanup_metal(node)
 
         self.debug("Waiting for the cluster to recover")
@@ -2902,7 +2944,7 @@ def __call__(self, node):
 
     def errorstoignore(self):
         '''Return list of errors which should be ignored'''
-        return [ """is running on remote1 which isn't allowed""",
+        return [ """is running on remote.*which isn't allowed""",
                  """Connection terminated""",
                  """Failed to send remote""",
                 ]
@@ -2918,6 +2960,7 @@ def __init__(self, cm):
         self.start = StartTest(cm)
         self.startall = SimulStartLite(cm)
         self.driver = RemoteDriver(cm)
+        self.is_docker_unsafe = 1
 
     def __call__(self, node):
         '''Perform the 'RemoteBaremetal' test. '''
@@ -2927,10 +2970,10 @@ def __call__(self, node):
         if not ret:
             return self.failure("Setup failed, start all nodes failed.")
 
-        self.driver.setup_env()
-        self.driver.step1_start_metal(node)
-        self.driver.step2_add_rsc(node)
-        self.driver.step3_test_attributes(node)
+        self.driver.setup_env(node)
+        self.driver.start_metal(node)
+        self.driver.add_dummy_rsc(node)
+        self.driver.test_attributes(node)
         self.driver.cleanup_metal(node)
 
         self.debug("Waiting for the cluster to recover")
@@ -2940,6 +2983,59 @@ def __call__(self, node):
 
         return self.success()
 
+    def is_applicable(self):
+        return self.driver.is_applicable()
+
+    def errorstoignore(self):
+        return self.driver.errorstoignore()
+
 AllTestClasses.append(RemoteBasic)
 
+###################################################################
+class RemoteStonithd(CTSTest):
+###################################################################
+    def __init__(self, cm):
+        CTSTest.__init__(self,cm)
+        self.name = "RemoteStonithd"
+        self.start = StartTest(cm)
+        self.startall = SimulStartLite(cm)
+        self.driver = RemoteDriver(cm)
+        self.is_docker_unsafe = 1
+
+    def __call__(self, node):
+        '''Perform the 'RemoteStonithd' test. '''
+        self.incr("calls")
+
+        ret = self.startall(None)
+        if not ret:
+            return self.failure("Setup failed, start all nodes failed.")
+
+        self.driver.setup_env(node)
+        self.driver.start_metal(node)
+        self.driver.add_dummy_rsc(node)
+
+        self.driver.fail_connection(node)
+        self.driver.cleanup_metal(node)
+
+        self.debug("Waiting for the cluster to recover")
+        self.CM.cluster_stable()
+        if self.driver.failed == 1:
+            return self.failure(self.driver.fail_string)
+
+        return self.success()
+
+    def is_applicable(self):
+        if not self.driver.is_applicable():
+            return False
+
+        if self.Env.has_key("DoFencing"):
+            return self.Env["DoFencing"]
+
+        return True
+
+    def errorstoignore(self):
+        return self.driver.errorstoignore()
+
+AllTestClasses.append(RemoteStonithd)
+
 # vim:ts=4:sw=4:et:

From c0f231e5fdd7ad30c0b1784db9d6a136d9472133 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Wed, 15 Oct 2014 17:12:51 -0400
Subject: [PATCH 24/29] Low: cts: baremetal remote stonithd  ignore patterns

---
 cts/CTStests.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/cts/CTStests.py b/cts/CTStests.py
index ae86e2ae390..e2653878296 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -3034,7 +3034,16 @@ def is_applicable(self):
         return True
 
     def errorstoignore(self):
-        return self.driver.errorstoignore()
+        ignore_pats = [
+            """Unexpected disconnect on remote-node""",
+            """error: process_lrm_event: Operation remote_.*_monitor""",
+            """LogActions: Recover remote_""",
+            """Calculated Transition .* /var/lib/pacemaker/pengine/pe-error""",
+            """error: native_create_actions: Resource .*ocf::.* is active on 2 nodes attempting recovery""",
+        ]
+
+        ignore_pats.extend(self.driver.errorstoignore())
+        return ignore_pats
 
 AllTestClasses.append(RemoteStonithd)
 

From 233c13d40ac261def30cc80519cffe8ff84dab7a Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Fri, 17 Oct 2014 11:54:19 -0400
Subject: [PATCH 25/29] Low: cts: Remote connection migration test

---
 cts/CTStests.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/cts/CTStests.py b/cts/CTStests.py
index e2653878296..55074e4c53f 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -2759,6 +2759,34 @@ def start_metal(self, node):
             self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
             self.failed = 1
 
+    def migrate_connection(self, node):
+        if self.failed == 1:
+            return
+
+        pats = [ ]
+        pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "migrate_to"))
+        pats.append(self.templates["Pat:RscOpOK"] % (self.remote_node, "migrate_from"))
+        pats.append(self.templates["Pat:DC_IDLE"])
+        watch = self.create_watch(pats, 120)
+        watch.setwatch()
+
+        (rc, lines) = self.rsh(node, "crm_resource -M -r %s" % (self.remote_node), None)
+        if rc != 0:
+            self.fail_string = "failed to move remote node connection resource"
+            self.logger.log(self.fail_string)
+            self.failed = 1
+            return
+
+        self.set_timer("remoteMetalMigrate")
+        watch.lookforall()
+        self.log_timer("remoteMetalMigrate")
+
+        if watch.unmatched:
+            self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
+            self.logger.log(self.fail_string)
+            self.failed = 1
+            return
+
     def fail_connection(self, node):
         if self.failed == 1:
             return
@@ -2876,6 +2904,7 @@ def cleanup_metal(self, node):
             self.rsh(node, "crm_resource -U -r %s -N %s" % (self.remote_rsc, self.remote_node))
             self.del_rsc(node, self.remote_rsc)
         if self.remote_node_added == 1:
+            self.rsh(node, "crm_resource -U -r %s" % (self.remote_node))
             self.del_rsc(node, self.remote_node)
         watch.lookforall()
         self.log_timer("remoteMetalCleanup")
@@ -3047,4 +3076,44 @@ def errorstoignore(self):
 
 AllTestClasses.append(RemoteStonithd)
 
+###################################################################
+class RemoteMigrate(CTSTest):
+###################################################################
+    def __init__(self, cm):
+        CTSTest.__init__(self,cm)
+        self.name = "RemoteMigrate"
+        self.start = StartTest(cm)
+        self.startall = SimulStartLite(cm)
+        self.driver = RemoteDriver(cm)
+        self.is_docker_unsafe = 1
+
+    def __call__(self, node):
+        '''Perform the 'RemoteMigrate' test. '''
+        self.incr("calls")
+
+        ret = self.startall(None)
+        if not ret:
+            return self.failure("Setup failed, start all nodes failed.")
+
+        self.driver.setup_env(node)
+        self.driver.start_metal(node)
+        self.driver.add_dummy_rsc(node)
+        self.driver.migrate_connection(node)
+        self.driver.cleanup_metal(node)
+
+        self.debug("Waiting for the cluster to recover")
+        self.CM.cluster_stable()
+        if self.driver.failed == 1:
+            return self.failure(self.driver.fail_string)
+
+        return self.success()
+
+    def is_applicable(self):
+        return self.driver.is_applicable()
+
+    def errorstoignore(self):
+        return self.driver.errorstoignore()
+
+AllTestClasses.append(RemoteMigrate)
+
 # vim:ts=4:sw=4:et:

From 1c32dd8083002326f7fd3f0f5d75cb9c0e57a66e Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Fri, 17 Oct 2014 15:16:01 -0400
Subject: [PATCH 26/29] Low: crm_resource: Allow -M location to be reinforced
 with -f even if rsc is already on node

---
 tools/crm_resource.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index e2da2442693..ff5effd05d2 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1980,6 +1980,7 @@ main(int argc, char **argv)
         node_t *current = NULL;
         node_t *dest = pe_find_node(data_set.nodes, host_uname);
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
+        gboolean cur_is_dest = FALSE;
 
         rc = -EINVAL;
 
@@ -2042,11 +2043,16 @@ main(int argc, char **argv)
 
         } else if(scope_master && rsc->fns->state(rsc, TRUE) != RSC_ROLE_MASTER) {
             crm_trace("%s is already active on %s but not in correct state", rsc_id, dest->details->uname);
-
         } else if (safe_str_eq(current->details->uname, dest->details->uname)) {
-            CMD_ERR("Error performing operation: %s is already %s on %s\n",
-                    rsc_id, scope_master?"promoted":"active", dest->details->uname);
-            goto bail;
+            cur_is_dest = TRUE;
+            if (do_force) {
+                crm_info("%s is already %s on %s, reinforcing placement with location constraint.\n",
+                         rsc_id, scope_master?"promoted":"active", dest->details->uname);
+            } else {
+                CMD_ERR("Error performing operation: %s is already %s on %s\n",
+                        rsc_id, scope_master?"promoted":"active", dest->details->uname);
+                goto bail;
+            }
         }
 
         /* Clear any previous constraints for 'dest' */
@@ -2058,7 +2064,10 @@ main(int argc, char **argv)
         crm_trace("%s%s now prefers node %s%s",
                   rsc->id, scope_master?" (master)":"", dest->details->uname, do_force?"(forced)":"");
 
-        if(do_force) {
+        /* only ban the previous location if current location != destination location.
+         * it is possible to use -M to enforce a location without regard of where the
+         * resource is currently located */
+        if(do_force && (cur_is_dest == FALSE)) {
             /* Ban the original location if possible */
             if(current) {
                 ban_resource(rsc_id, current->details->uname, NULL, cib_conn);

From f06b21bd52a43b1ecda02b0ef3696a4e67ead4c5 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Fri, 17 Oct 2014 18:32:27 -0400
Subject: [PATCH 27/29] Low: cts: RemoteRscFailure test verifies monitoring of
 rscs on remote-nodes

---
 cts/CTStests.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 83 insertions(+), 5 deletions(-)

diff --git a/cts/CTStests.py b/cts/CTStests.py
index 55074e4c53f..7bca0fb9296 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -2678,14 +2678,12 @@ def add_rsc(self, node, rsc_xml):
 
     def add_primitive_rsc(self, node):
         rsc_xml = """
-<primitive class="ocf" id="%s" provider="pacemaker" type="Dummy">
+<primitive class="ocf" id="%s" provider="heartbeat" type="Dummy">
     <operations>
       <op id="remote-rsc-monitor-interval-10s" interval="10s" name="monitor"/>
     </operations>
-    <meta_attributes id="remote-meta_attributes">
-          <nvpair id="remote-meta-container" name="container" value="%s"/>
-    </meta_attributes id="remote-meta_attributes">
-</primitive>""" % (self.remote_rsc, self.remote_node)
+    <meta_attributes id="remote-meta_attributes"/>
+</primitive>""" % (self.remote_rsc)
         self.add_rsc(node, rsc_xml)
         if self.failed == 0:
             self.remote_rsc_added = 1
@@ -2787,6 +2785,30 @@ def migrate_connection(self, node):
             self.failed = 1
             return
 
+    def fail_rsc(self, node):
+        if self.failed == 1:
+            return
+
+        watchpats = [ ]
+        watchpats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "stop", self.remote_node))
+        watchpats.append(self.templates["Pat:RscRemoteOpOK"] % (self.remote_rsc, "start", self.remote_node))
+        watchpats.append(self.templates["Pat:DC_IDLE"])
+
+        watch = self.create_watch(watchpats, 120)
+        watch.setwatch()
+
+        self.debug("causing dummy rsc to fail.")
+
+        rc = self.rsh(node, "rm -f /var/run/resource-agents/Dummy*")
+
+        self.set_timer("remoteRscFail")
+        watch.lookforall()
+        self.log_timer("remoteRscFail")
+        if watch.unmatched:
+            self.fail_string = "Unmatched patterns during rsc fail: %s" % (repr(watch.unmatched))
+            self.logger.log(self.fail_string)
+            self.failed = 1
+
     def fail_connection(self, node):
         if self.failed == 1:
             return
@@ -2854,6 +2876,13 @@ def add_dummy_rsc(self, node):
         # Add a resource that must live on remote-node
         self.add_primitive_rsc(node)
 
+        # force that rsc to prefer the remote node. 
+        (rc, line) = self.CM.rsh(node, "crm_resource -M -r %s -N %s -f" % (self.remote_rsc, self.remote_node), None)
+        if rc != 0:
+            self.fail_string = "Failed to place remote resource on remote node."
+            self.failed = 1
+            return
+
         self.set_timer("remoteMetalRsc")
         watch.lookforall()
         self.log_timer("remoteMetalRsc")
@@ -3116,4 +3145,53 @@ def errorstoignore(self):
 
 AllTestClasses.append(RemoteMigrate)
 
+
+###################################################################
+class RemoteRscFailure(CTSTest):
+###################################################################
+    def __init__(self, cm):
+
+        # fail a rsc on a remote node, verify recovery.
+        CTSTest.__init__(self,cm)
+        self.name = "RemoteRscFailure"
+        self.start = StartTest(cm)
+        self.startall = SimulStartLite(cm)
+        self.driver = RemoteDriver(cm)
+        self.is_docker_unsafe = 1
+
+    def __call__(self, node):
+        '''Perform the 'RemoteRscFailure' test. '''
+        self.incr("calls")
+
+        ret = self.startall(None)
+        if not ret:
+            return self.failure("Setup failed, start all nodes failed.")
+
+        self.driver.setup_env(node)
+        self.driver.start_metal(node)
+        self.driver.add_dummy_rsc(node)
+
+        # This is an important step. We are migrating the connection
+        # before failing the resource. This verifies that the migration
+        # has properly maintained control over the remote-node.
+        self.driver.migrate_connection(node)
+
+        self.driver.fail_rsc(node)
+        self.driver.cleanup_metal(node)
+
+        self.debug("Waiting for the cluster to recover")
+        self.CM.cluster_stable()
+        if self.driver.failed == 1:
+            return self.failure(self.driver.fail_string)
+
+        return self.success()
+
+    def is_applicable(self):
+        return self.driver.is_applicable()
+
+    def errorstoignore(self):
+        return self.driver.errorstoignore()
+
+AllTestClasses.append(RemoteRscFailure)
+
 # vim:ts=4:sw=4:et:

From 31867a4d4d16b47e7ee4a23cb4f073631ba82337 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Tue, 21 Oct 2014 15:43:01 -0400
Subject: [PATCH 28/29] Low: cts: ignore patterns for RemoteRscFailure test

---
 cts/CTStests.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cts/CTStests.py b/cts/CTStests.py
index 7bca0fb9296..d2b7668ac3e 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -3190,7 +3190,12 @@ def is_applicable(self):
         return self.driver.is_applicable()
 
     def errorstoignore(self):
-        return self.driver.errorstoignore()
+        ignore_pats = [
+            """LogActions: Recover remote-rsc""",
+        ]
+
+        ignore_pats.extend(self.driver.errorstoignore())
+        return ignore_pats
 
 AllTestClasses.append(RemoteRscFailure)
 

From 1b855f9a627539e75007e40bce4c79f2f2a60579 Mon Sep 17 00:00:00 2001
From: David Vossel <dvossel@redhat.com>
Date: Tue, 21 Oct 2014 17:18:30 -0400
Subject: [PATCH 29/29] Low: build: Add ability to set custom spec version for
 'make rpm'

---
 GNUmakefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/GNUmakefile b/GNUmakefile
index b17fb4cb901..f28dea88397 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -58,6 +58,8 @@ BUILD_COUNTER	?= build.counter
 LAST_COUNT      = $(shell test ! -e $(BUILD_COUNTER) && echo 0; test -e $(BUILD_COUNTER) && cat $(BUILD_COUNTER))
 COUNT           = $(shell expr 1 + $(LAST_COUNT))
 
+SPECVERSION	?= $(COUNT)
+
 init:
 	./autogen.sh
 
@@ -144,7 +146,7 @@ srpm-%:	export $(PACKAGE)-%.spec
 	if [ -e $(BUILD_COUNTER) ]; then					\
 		echo $(COUNT) > $(BUILD_COUNTER);				\
 	fi
-	sed -i 's/global\ specversion.*/global\ specversion\ $(COUNT)/' $(PACKAGE).spec
+	sed -i 's/global\ specversion.*/global\ specversion\ $(SPECVERSION)/' $(PACKAGE).spec
 	sed -i 's/global\ commit.*/global\ commit\ $(TAG)/' $(PACKAGE).spec
 	case "$(WITH)" in 	\
 	  *pre_release*)	\