Merge pull request #595 from gao-yan/master-asymmetrical-order

Fix: pengine: Fix the behaviors of multi-state resources with asymmetrical ordering
ClusterLabs · Oct 27, 2014 · acd5da7 · acd5da7
2 parents 7f40572 + fccaa46
commit acd5da7
Show file tree

Hide file tree

Showing 7 changed files with 279 additions and 1 deletion.
diff --git a/pengine/native.c b/pengine/native.c
@@ -1889,7 +1889,7 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_
         } else if ((then_rsc_role == RSC_ROLE_STOPPED) && safe_str_eq(then->task, RSC_STOP)) {
             /* ignore... if 'then' is supposed to be stopped after 'first', but
              * then is already stopped, there is nothing to be done when non-symmetrical.  */
-        } else if ((then_rsc_role == RSC_ROLE_STARTED) && safe_str_eq(then->task, RSC_START)) {
+        } else if ((then_rsc_role >= RSC_ROLE_STARTED) && safe_str_eq(then->task, RSC_START)) {
             /* ignore... if 'then' is supposed to be started after 'first', but
              * then is already started, there is nothing to be done when non-symmetrical.  */
         } else if (!(first->flags & pe_action_runnable)) {

diff --git a/pengine/regression.sh b/pengine/regression.sh
@@ -396,6 +396,7 @@ do_test master-stop "Stop instances due to location constraint with role=Started
 do_test master-partially-demoted-group "Allow partially demoted group to finish demoting"
 do_test bug-cl-5213 "Ensure role colocation with -INFINITY is enforced"
 do_test bug-cl-5219 "Allow unrelated resources with a common colocation target to remain promoted"
+do_test master-asymmetrical-order "Fix the behaviors of multi-state resources with asymmetrical ordering"
 
 echo ""
 do_test history-1 "Correctly parse stateful-1 resource state"

diff --git a/pengine/test10/master-asymmetrical-order.dot b/pengine/test10/master-asymmetrical-order.dot
@@ -0,0 +1,24 @@
+digraph "g" {
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"ms1_demote_0" -> "ms1_demoted_0" [ style = bold]
+"ms1_demote_0" -> "rsc1:0_demote_0 node1" [ style = bold]
+"ms1_demote_0" [ style=bold color="green" fontcolor="orange"]
+"ms1_demoted_0" -> "ms1_stop_0" [ style = bold]
+"ms1_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"ms1_stop_0" -> "ms1_stopped_0" [ style = bold]
+"ms1_stop_0" -> "rsc1:0_stop_0 node1" [ style = bold]
+"ms1_stop_0" -> "rsc1:1_stop_0 node2" [ style = bold]
+"ms1_stop_0" [ style=bold color="green" fontcolor="orange"]
+"ms1_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"probe_complete node1" [ style=bold color="green" fontcolor="black"]
+"probe_complete node2" [ style=bold color="green" fontcolor="black"]
+"rsc1:0_demote_0 node1" -> "ms1_demoted_0" [ style = bold]
+"rsc1:0_demote_0 node1" -> "rsc1:0_stop_0 node1" [ style = bold]
+"rsc1:0_demote_0 node1" [ style=bold color="green" fontcolor="black"]
+"rsc1:0_stop_0 node1" -> "all_stopped" [ style = bold]
+"rsc1:0_stop_0 node1" -> "ms1_stopped_0" [ style = bold]
+"rsc1:0_stop_0 node1" [ style=bold color="green" fontcolor="black"]
+"rsc1:1_stop_0 node2" -> "all_stopped" [ style = bold]
+"rsc1:1_stop_0 node2" -> "ms1_stopped_0" [ style = bold]
+"rsc1:1_stop_0 node2" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/master-asymmetrical-order.exp b/pengine/test10/master-asymmetrical-order.exp
@@ -0,0 +1,128 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="6" operation="stop" operation_key="rsc1:0_stop_0" on_node="node1" on_node_uuid="node1">
+        <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="5" operation="demote" operation_key="rsc1:0_demote_0" on_node="node1" on_node_uuid="node1"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="10" operation="stop" operation_key="ms1_stop_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="5" operation="demote" operation_key="rsc1:0_demote_0" on_node="node1" on_node_uuid="node1">
+        <primitive id="rsc1" long-id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful"/>
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="14" operation="demote" operation_key="ms1_demote_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="2">
+    <action_set>
+      <rsc_op id="7" operation="stop" operation_key="rsc1:1_stop_0" on_node="node2" on_node_uuid="node2">
+        <primitive id="rsc1" long-id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="10" operation="stop" operation_key="ms1_stop_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="3" priority="1000000">
+    <action_set>
+      <pseudo_event id="15" operation="demoted" operation_key="ms1_demoted_0">
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="5" operation="demote" operation_key="rsc1:0_demote_0" on_node="node1" on_node_uuid="node1"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="14" operation="demote" operation_key="ms1_demote_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="4">
+    <action_set>
+      <pseudo_event id="14" operation="demote" operation_key="ms1_demote_0">
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="5" priority="1000000">
+    <action_set>
+      <pseudo_event id="11" operation="stopped" operation_key="ms1_stopped_0">
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="6" operation="stop" operation_key="rsc1:0_stop_0" on_node="node1" on_node_uuid="node1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="7" operation="stop" operation_key="rsc1:1_stop_0" on_node="node2" on_node_uuid="node2"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="10" operation="stop" operation_key="ms1_stop_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="6">
+    <action_set>
+      <pseudo_event id="10" operation="stop" operation_key="ms1_stop_0">
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="15" operation="demoted" operation_key="ms1_demoted_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="7" priority="1000000">
+    <action_set>
+      <rsc_op id="4" operation="probe_complete" operation_key="probe_complete-node2" on_node="node2" on_node_uuid="node2">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="8" priority="1000000">
+    <action_set>
+      <rsc_op id="3" operation="probe_complete" operation_key="probe_complete-node1" on_node="node1" on_node_uuid="node1">
+        <attributes CRM_meta_op_no_wait="true" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="9">
+    <action_set>
+      <pseudo_event id="1" operation="all_stopped" operation_key="all_stopped">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="6" operation="stop" operation_key="rsc1:0_stop_0" on_node="node1" on_node_uuid="node1"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="7" operation="stop" operation_key="rsc1:1_stop_0" on_node="node2" on_node_uuid="node2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+</transition_graph>
diff --git a/pengine/test10/master-asymmetrical-order.scores b/pengine/test10/master-asymmetrical-order.scores
@@ -0,0 +1,25 @@
+Allocation scores:
+clone_color: ms1 allocation score on node1: 0
+clone_color: ms1 allocation score on node2: 0
+clone_color: ms2 allocation score on node1: 0
+clone_color: ms2 allocation score on node2: 0
+clone_color: rsc1:0 allocation score on node1: 1
+clone_color: rsc1:0 allocation score on node2: 0
+clone_color: rsc1:1 allocation score on node1: 0
+clone_color: rsc1:1 allocation score on node2: 1
+clone_color: rsc2:0 allocation score on node1: 1
+clone_color: rsc2:0 allocation score on node2: 0
+clone_color: rsc2:1 allocation score on node1: 0
+clone_color: rsc2:1 allocation score on node2: 1
+native_color: rsc1:0 allocation score on node1: -INFINITY
+native_color: rsc1:0 allocation score on node2: -INFINITY
+native_color: rsc1:1 allocation score on node1: -INFINITY
+native_color: rsc1:1 allocation score on node2: -INFINITY
+native_color: rsc2:0 allocation score on node1: 1
+native_color: rsc2:0 allocation score on node2: 0
+native_color: rsc2:1 allocation score on node1: -INFINITY
+native_color: rsc2:1 allocation score on node2: 1
+rsc1:0 promotion score on none: 0
+rsc1:1 promotion score on none: 0
+rsc2:0 promotion score on node1: -1
+rsc2:1 promotion score on node2: 99
diff --git a/pengine/test10/master-asymmetrical-order.summary b/pengine/test10/master-asymmetrical-order.summary
@@ -0,0 +1,34 @@
+
+Current cluster status:
+Online: [ node1 node2 ]
+
+ Master/Slave Set: ms1 [rsc1]
+     Masters: [ node1 ]
+     Slaves: [ node2 ]
+ Master/Slave Set: ms2 [rsc2]
+     Masters: [ node2 ]
+     Slaves: [ node1 ]
+
+Transition Summary:
+ * Demote  rsc1:0	(Master -> Stopped node1)
+ * Stop    rsc1:1	(node2)
+
+Executing cluster transition:
+ * Pseudo action:   ms1_demote_0
+ * Resource action: rsc1:0          demote on node1
+ * Pseudo action:   ms1_demoted_0
+ * Pseudo action:   ms1_stop_0
+ * Resource action: rsc1:0          stop on node1
+ * Resource action: rsc1:1          stop on node2
+ * Pseudo action:   ms1_stopped_0
+ * Pseudo action:   all_stopped
+
+Revised cluster status:
+Online: [ node1 node2 ]
+
+ Master/Slave Set: ms1 [rsc1]
+     Stopped: [ node1 node2 ]
+ Master/Slave Set: ms2 [rsc2]
+     Masters: [ node2 ]
+     Slaves: [ node1 ]
+
diff --git a/pengine/test10/master-asymmetrical-order.xml b/pengine/test10/master-asymmetrical-order.xml
@@ -0,0 +1,66 @@
+<cib epoch="1" num_updates="30" admin_epoch="0" validate-with="pacemaker-1.2" cib-last-written="Fri Jul 13 13:51:19 2012" have-quorum="1">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
+        <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="node1" type="normal" uname="node1"/>
+      <node id="node2" type="normal" uname="node2"/>
+    </nodes>
+    <resources>
+      <master id="ms1">
+        <primitive id="rsc1" class="ocf" provider="pacemaker" type="Stateful">
+          <operations id="rsc1-operations">
+            <op id="rsc1-monitor-master-5" interval="5" name="monitor" role="Master"/>
+            <op id="rsc1-monitor-slave-5" interval="5" name="monitor" role="Slave"/>
+          </operations>
+        </primitive>
+        <meta_attributes id="ms1-meta_attributes">
+          <nvpair id="ms1-meta_attributes-target-role" name="target-role" value="Stopped"/>
+        </meta_attributes>
+      </master>
+      <master id="ms2">
+        <primitive id="rsc2" class="ocf" provider="pacemaker" type="Stateful">
+          <operations id="rsc2-operations">
+            <op id="rsc2-monitor-master-5" interval="5" name="monitor" role="Master"/>
+            <op id="rsc2-monitor-slave-5" interval="5" name="monitor" role="Slave"/>
+          </operations>
+        </primitive>
+      </master>
+    </resources>
+    <constraints>
+      <rsc_location id="ms1-master-location" rsc="ms1" role="Master" node="node1" score="100"/>
+      <rsc_location id="ms2-master-location" rsc="ms2" role="Master" node="node2" score="100"/>
+      <rsc_order id="order-ms1-ms2" first="ms1" first-action="promote" then="ms2" then-action="start" kind="Mandatory" symmetrical="false"/>
+    </constraints>
+  </configuration>
+  <status>
+    <node_state id="node1" uname="node1" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
+      <lrm id="node1">
+        <lrm_resources>
+          <lrm_resource id="rsc1:0" class="ocf" provider="pacemaker" type="Stateful">
+            <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.9" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1412938661" last-rc-change="1412938661" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="rsc2:0" class="ocf" provider="pacemaker" type="Stateful">
+            <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.9" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1412938661" last-rc-change="1412938661" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="node2" uname="node2" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
+      <lrm id="node2">
+        <lrm_resources>
+          <lrm_resource id="rsc1:1" class="ocf" provider="pacemaker" type="Stateful">
+            <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.9" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1412938661" last-rc-change="1412938661" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="rsc2:1" class="ocf" provider="pacemaker" type="Stateful">
+            <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.9" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1412938661" last-rc-change="1412938661" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+  </status>
+</cib>