Skip to content

Commit

Permalink
Merge pull request #1398 from kgaillot/fixes
Browse files Browse the repository at this point in the history 
Recover after failed demote when appropriate
  • Loading branch information
@kgaillot
kgaillot committed Jan 15, 2018
2 parents 26ebf94 + 6f89cd6 commit 0d8c767
Show file tree
Hide file tree
Showing 12 changed files with 1,315 additions and 7 deletions.
13 changes: 6 additions & 7 deletions lib/pengine/unpack.c
View file
Expand Up @@ -2662,11 +2662,6 @@ unpack_rsc_op_failure(resource_t * rsc, node_t * node, int rc, xmlNode * xml_op,
rsc->role = RSC_ROLE_MASTER;

} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
/*
* staying in role=master ends up putting the PE/TE into a loop
* setting role=slave is not dangerous because no master will be
* promoted until the failed resource has been fully stopped
*/
if (action->on_fail == action_fail_block) {
rsc->role = RSC_ROLE_MASTER;
rsc->next_role = RSC_ROLE_STOPPED;
Expand All @@ -2675,9 +2670,13 @@ unpack_rsc_op_failure(resource_t * rsc, node_t * node, int rc, xmlNode * xml_op,
rsc->role = RSC_ROLE_STOPPED;

} else {
crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
/*
* Staying in master role would put the PE/TE into a loop. Setting
* slave role is not dangerous because the resource will be stopped
* as part of recovery, and any master promotion will be ordered
* after that stop.
*/
rsc->role = RSC_ROLE_SLAVE;
rsc->next_role = RSC_ROLE_STOPPED;
}

} else if (compare_version("2.0", op_version) > 0 && safe_str_eq(task, CRMD_ACTION_START)) {
Expand Down
2 changes: 2 additions & 0 deletions pengine/regression.sh
View file
Expand Up @@ -447,6 +447,8 @@ do_test bug-cl-5219 "Allow unrelated resources with a common colocation target t
do_test master-asymmetrical-order "Fix the behaviors of multi-state resources with asymmetrical ordering"
do_test master-notify "Master promotion with notifies"
do_test master-score-startup "Use permanent master scores without LRM history"
do_test failed-demote-recovery "Recover resource in slave role after demote fails"
do_test failed-demote-recovery-master "Recover resource in master role after demote fails"

echo ""
do_test history-1 "Correctly parse stateful-1 resource state"
Expand Down
92 changes: 92 additions & 0 deletions pengine/test10/failed-demote-recovery-master.dot
View file
@@ -0,0 +1,92 @@
digraph "g" {
"DB2_HADR-master_confirmed-post_notify_promoted_0" -> "DB2_HADR_monitor_22000 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_confirmed-post_notify_running_0" -> "DB2_HADR-master_pre_notify_promote_0" [ style = bold]
"DB2_HADR-master_confirmed-post_notify_running_0" -> "DB2_HADR_monitor_22000 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_confirmed-post_notify_stopped_0" -> "DB2_HADR-master_pre_notify_promote_0" [ style = bold]
"DB2_HADR-master_confirmed-post_notify_stopped_0" -> "DB2_HADR-master_pre_notify_start_0" [ style = bold]
"DB2_HADR-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold]
"DB2_HADR-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_confirmed-pre_notify_promote_0" -> "DB2_HADR-master_post_notify_promoted_0" [ style = bold]
"DB2_HADR-master_confirmed-pre_notify_promote_0" -> "DB2_HADR-master_promote_0" [ style = bold]
"DB2_HADR-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_confirmed-pre_notify_start_0" -> "DB2_HADR-master_post_notify_running_0" [ style = bold]
"DB2_HADR-master_confirmed-pre_notify_start_0" -> "DB2_HADR-master_start_0" [ style = bold]
"DB2_HADR-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_confirmed-pre_notify_stop_0" -> "DB2_HADR-master_post_notify_stopped_0" [ style = bold]
"DB2_HADR-master_confirmed-pre_notify_stop_0" -> "DB2_HADR-master_stop_0" [ style = bold]
"DB2_HADR-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_post_notify_promoted_0" -> "DB2_HADR-master_confirmed-post_notify_promoted_0" [ style = bold]
"DB2_HADR-master_post_notify_promoted_0" -> "DB2_HADR_post_notify_promoted_0 fastvm-rhel-7-4-95" [ style = bold]
"DB2_HADR-master_post_notify_promoted_0" -> "DB2_HADR_post_notify_promoted_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_post_notify_running_0" -> "DB2_HADR-master_confirmed-post_notify_running_0" [ style = bold]
"DB2_HADR-master_post_notify_running_0" -> "DB2_HADR_post_notify_running_0 fastvm-rhel-7-4-95" [ style = bold]
"DB2_HADR-master_post_notify_running_0" -> "DB2_HADR_post_notify_running_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_post_notify_stopped_0" -> "DB2_HADR-master_confirmed-post_notify_stopped_0" [ style = bold]
"DB2_HADR-master_post_notify_stopped_0" -> "DB2_HADR_post_notify_stopped_0 fastvm-rhel-7-4-95" [ style = bold]
"DB2_HADR-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_pre_notify_promote_0" -> "DB2_HADR-master_confirmed-pre_notify_promote_0" [ style = bold]
"DB2_HADR-master_pre_notify_promote_0" -> "DB2_HADR_pre_notify_promote_0 fastvm-rhel-7-4-95" [ style = bold]
"DB2_HADR-master_pre_notify_promote_0" -> "DB2_HADR_pre_notify_promote_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_pre_notify_start_0" -> "DB2_HADR-master_confirmed-pre_notify_start_0" [ style = bold]
"DB2_HADR-master_pre_notify_start_0" -> "DB2_HADR_pre_notify_start_0 fastvm-rhel-7-4-95" [ style = bold]
"DB2_HADR-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_pre_notify_stop_0" -> "DB2_HADR-master_confirmed-pre_notify_stop_0" [ style = bold]
"DB2_HADR-master_pre_notify_stop_0" -> "DB2_HADR_pre_notify_stop_0 fastvm-rhel-7-4-95" [ style = bold]
"DB2_HADR-master_pre_notify_stop_0" -> "DB2_HADR_pre_notify_stop_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_promote_0" -> "DB2_HADR_promote_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_promote_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_promoted_0" -> "DB2_HADR-master_post_notify_promoted_0" [ style = bold]
"DB2_HADR-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_running_0" -> "DB2_HADR-master_post_notify_running_0" [ style = bold]
"DB2_HADR-master_running_0" -> "DB2_HADR-master_promote_0" [ style = bold]
"DB2_HADR-master_running_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_start_0" -> "DB2_HADR-master_running_0" [ style = bold]
"DB2_HADR-master_start_0" -> "DB2_HADR_start_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_start_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_stop_0" -> "DB2_HADR-master_stopped_0" [ style = bold]
"DB2_HADR-master_stop_0" -> "DB2_HADR_stop_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR-master_stopped_0" -> "DB2_HADR-master_post_notify_stopped_0" [ style = bold]
"DB2_HADR-master_stopped_0" -> "DB2_HADR-master_promote_0" [ style = bold]
"DB2_HADR-master_stopped_0" -> "DB2_HADR-master_start_0" [ style = bold]
"DB2_HADR-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"DB2_HADR_monitor_22000 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_post_notify_promoted_0 fastvm-rhel-7-4-95" -> "DB2_HADR-master_confirmed-post_notify_promoted_0" [ style = bold]
"DB2_HADR_post_notify_promoted_0 fastvm-rhel-7-4-95" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_post_notify_promoted_0 fastvm-rhel-7-4-96" -> "DB2_HADR-master_confirmed-post_notify_promoted_0" [ style = bold]
"DB2_HADR_post_notify_promoted_0 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_post_notify_running_0 fastvm-rhel-7-4-95" -> "DB2_HADR-master_confirmed-post_notify_running_0" [ style = bold]
"DB2_HADR_post_notify_running_0 fastvm-rhel-7-4-95" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_post_notify_running_0 fastvm-rhel-7-4-96" -> "DB2_HADR-master_confirmed-post_notify_running_0" [ style = bold]
"DB2_HADR_post_notify_running_0 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_post_notify_stopped_0 fastvm-rhel-7-4-95" -> "DB2_HADR-master_confirmed-post_notify_stopped_0" [ style = bold]
"DB2_HADR_post_notify_stopped_0 fastvm-rhel-7-4-95" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_pre_notify_promote_0 fastvm-rhel-7-4-95" -> "DB2_HADR-master_confirmed-pre_notify_promote_0" [ style = bold]
"DB2_HADR_pre_notify_promote_0 fastvm-rhel-7-4-95" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_pre_notify_promote_0 fastvm-rhel-7-4-96" -> "DB2_HADR-master_confirmed-pre_notify_promote_0" [ style = bold]
"DB2_HADR_pre_notify_promote_0 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_pre_notify_start_0 fastvm-rhel-7-4-95" -> "DB2_HADR-master_confirmed-pre_notify_start_0" [ style = bold]
"DB2_HADR_pre_notify_start_0 fastvm-rhel-7-4-95" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_pre_notify_stop_0 fastvm-rhel-7-4-95" -> "DB2_HADR-master_confirmed-pre_notify_stop_0" [ style = bold]
"DB2_HADR_pre_notify_stop_0 fastvm-rhel-7-4-95" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_pre_notify_stop_0 fastvm-rhel-7-4-96" -> "DB2_HADR-master_confirmed-pre_notify_stop_0" [ style = bold]
"DB2_HADR_pre_notify_stop_0 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_promote_0 fastvm-rhel-7-4-96" -> "DB2_HADR-master_promoted_0" [ style = bold]
"DB2_HADR_promote_0 fastvm-rhel-7-4-96" -> "DB2_HADR_monitor_22000 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR_promote_0 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_start_0 fastvm-rhel-7-4-96" -> "DB2_HADR-master_running_0" [ style = bold]
"DB2_HADR_start_0 fastvm-rhel-7-4-96" -> "DB2_HADR_monitor_22000 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR_start_0 fastvm-rhel-7-4-96" -> "DB2_HADR_promote_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR_start_0 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"DB2_HADR_stop_0 fastvm-rhel-7-4-96" -> "DB2_HADR-master_stopped_0" [ style = bold]
"DB2_HADR_stop_0 fastvm-rhel-7-4-96" -> "DB2_HADR_start_0 fastvm-rhel-7-4-96" [ style = bold]
"DB2_HADR_stop_0 fastvm-rhel-7-4-96" -> "all_stopped" [ style = bold]
"DB2_HADR_stop_0 fastvm-rhel-7-4-96" [ style=bold color="green" fontcolor="black"]
"all_stopped" [ style=bold color="green" fontcolor="orange"]
}

0 comments on commit 0d8c767

Please sign in to comment.