Skip to content

Commit

Permalink
Fix: libpacemaker: Don't shuffle anonymous clone instances
Browse files Browse the repository at this point in the history 
Currently, anonymous clone instances may be shuffled under certain
conditions, causing an unnecessary resource downtime when an instance is
moved away from its current running node.

For example, this can happen when a stopped promotable instance is
scheduled to promote and the stickiness is lower than the promotion
score (see the promotable-anon-recover-promoted test). Instance 0 gets
allocated first and goes to the node that will be promoted, causing it
to relocate if it's already running somewhere else.

There are also some other corner cases that can trigger shuffling, like
the one in the clone-anon-no-shuffle-constraints test.

The fix is to wait until all instance allocations are finished, and then
to swap allocations to keep as many instances as possible on their
current running nodes.

This commit also updates several tests that contain unnecessary instance
moves.

Resolves: RHBZ#1931023

Signed-off-by: Reid Wahl <nrwahl@protonmail.com>
  • Loading branch information
@nrwahl2
nrwahl2 committed Mar 22, 2021
1 parent 4ab0c12 commit f10da99
Show file tree
Hide file tree
Showing 15 changed files with 573 additions and 1,270 deletions.
99 changes: 15 additions & 84 deletions cts/scheduler/dot/cancel-behind-moving-remote.dot
View file

Large diffs are not rendered by default.

43 changes: 0 additions & 43 deletions cts/scheduler/dot/clone-anon-no-shuffle-constraints.dot
View file
Original file line number Diff line number Diff line change
@@ -1,47 +1,4 @@
digraph "g" {
"SASHOME-clone_running_0" [ style=bold color="green" fontcolor="orange"]
"SASHOME-clone_start_0" -> "SASHOME-clone_running_0" [ style = bold]
"SASHOME-clone_start_0" -> "SASHOME_start_0 node1" [ style = bold]
"SASHOME-clone_start_0" -> "SASHOME_start_0 node6" [ style = bold]
"SASHOME-clone_start_0" -> "SASHOME_start_0 node8" [ style = bold]
"SASHOME-clone_start_0" -> "SASHOME_start_0 node9" [ style = bold]
"SASHOME-clone_start_0" [ style=bold color="green" fontcolor="orange"]
"SASHOME-clone_stop_0" -> "SASHOME-clone_stopped_0" [ style = bold]
"SASHOME-clone_stop_0" -> "SASHOME_stop_0 node1" [ style = bold]
"SASHOME-clone_stop_0" -> "SASHOME_stop_0 node6" [ style = bold]
"SASHOME-clone_stop_0" -> "SASHOME_stop_0 node8" [ style = bold]
"SASHOME-clone_stop_0" -> "SASHOME_stop_0 node9" [ style = bold]
"SASHOME-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"SASHOME-clone_stopped_0" -> "SASHOME-clone_start_0" [ style = bold]
"SASHOME-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"SASHOME_monitor_10000 node1" [ style=bold color="green" fontcolor="black"]
"SASHOME_monitor_10000 node6" [ style=bold color="green" fontcolor="black"]
"SASHOME_monitor_10000 node8" [ style=bold color="green" fontcolor="black"]
"SASHOME_monitor_10000 node9" [ style=bold color="green" fontcolor="black"]
"SASHOME_start_0 node1" -> "SASHOME-clone_running_0" [ style = bold]
"SASHOME_start_0 node1" -> "SASHOME_monitor_10000 node1" [ style = bold]
"SASHOME_start_0 node1" [ style=bold color="green" fontcolor="black"]
"SASHOME_start_0 node6" -> "SASHOME-clone_running_0" [ style = bold]
"SASHOME_start_0 node6" -> "SASHOME_monitor_10000 node6" [ style = bold]
"SASHOME_start_0 node6" [ style=bold color="green" fontcolor="black"]
"SASHOME_start_0 node8" -> "SASHOME-clone_running_0" [ style = bold]
"SASHOME_start_0 node8" -> "SASHOME_monitor_10000 node8" [ style = bold]
"SASHOME_start_0 node8" [ style=bold color="green" fontcolor="black"]
"SASHOME_start_0 node9" -> "SASHOME-clone_running_0" [ style = bold]
"SASHOME_start_0 node9" -> "SASHOME_monitor_10000 node9" [ style = bold]
"SASHOME_start_0 node9" [ style=bold color="green" fontcolor="black"]
"SASHOME_stop_0 node1" -> "SASHOME-clone_stopped_0" [ style = bold]
"SASHOME_stop_0 node1" -> "SASHOME_start_0 node9" [ style = bold]
"SASHOME_stop_0 node1" [ style=bold color="green" fontcolor="black"]
"SASHOME_stop_0 node6" -> "SASHOME-clone_stopped_0" [ style = bold]
"SASHOME_stop_0 node6" -> "SASHOME_start_0 node1" [ style = bold]
"SASHOME_stop_0 node6" [ style=bold color="green" fontcolor="black"]
"SASHOME_stop_0 node8" -> "SASHOME-clone_stopped_0" [ style = bold]
"SASHOME_stop_0 node8" -> "SASHOME_start_0 node6" [ style = bold]
"SASHOME_stop_0 node8" [ style=bold color="green" fontcolor="black"]
"SASHOME_stop_0 node9" -> "SASHOME-clone_stopped_0" [ style = bold]
"SASHOME_stop_0 node9" -> "SASHOME_start_0 node8" [ style = bold]
"SASHOME_stop_0 node9" [ style=bold color="green" fontcolor="black"]
"SAS_PROD_FENCE_monitor_60000 node1" [ style=bold color="green" fontcolor="black"]
"SAS_PROD_FENCE_start_0 node1" -> "SAS_PROD_FENCE_monitor_60000 node1" [ style = bold]
"SAS_PROD_FENCE_start_0 node1" [ style=bold color="green" fontcolor="black"]
Expand Down
44 changes: 22 additions & 22 deletions cts/scheduler/dot/novell-252693-3.dot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,28 +72,28 @@
"evmsdcloneset_start_0" -> "evmsdclone:1_start_0 node1" [ style = bold]
"evmsdcloneset_start_0" -> "evmsdcloneset_running_0" [ style = bold]
"evmsdcloneset_start_0" [ style=bold color="green" fontcolor="orange"]
"imagestoreclone:0_monitor_0 node1" -> "imagestorecloneset_start_0" [ style = bold]
"imagestoreclone:0_monitor_0 node1" -> "imagestorecloneset_stopped_0" [ style = bold]
"imagestoreclone:0_monitor_0 node1" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_monitor_20000 node1" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_post_notify_start_0 node1" -> "imagestorecloneset_confirmed-post_notify_running_0" [ style = bold]
"imagestoreclone:0_post_notify_start_0 node1" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_monitor_20000 node2" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_post_notify_start_0 node2" -> "imagestorecloneset_confirmed-post_notify_running_0" [ style = bold]
"imagestoreclone:0_post_notify_start_0 node2" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_pre_notify_stop_0 node2" -> "imagestorecloneset_confirmed-pre_notify_stop_0" [ style = bold]
"imagestoreclone:0_pre_notify_stop_0 node2" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_start_0 node1" -> "imagestoreclone:0_monitor_20000 node1" [ style = bold]
"imagestoreclone:0_start_0 node1" -> "imagestorecloneset_running_0" [ style = bold]
"imagestoreclone:0_start_0 node1" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_stop_0 node2" -> "imagestoreclone:0_start_0 node1" [ style = bold]
"imagestoreclone:0_start_0 node2" -> "imagestoreclone:0_monitor_20000 node2" [ style = bold]
"imagestoreclone:0_start_0 node2" -> "imagestorecloneset_running_0" [ style = bold]
"imagestoreclone:0_start_0 node2" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:0_stop_0 node2" -> "imagestoreclone:0_start_0 node2" [ style = bold]
"imagestoreclone:0_stop_0 node2" -> "imagestorecloneset_stopped_0" [ style = bold]
"imagestoreclone:0_stop_0 node2" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:1_monitor_20000 node2" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:1_post_notify_start_0 node2" -> "imagestorecloneset_confirmed-post_notify_running_0" [ style = bold]
"imagestoreclone:1_post_notify_start_0 node2" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:1_start_0 node2" -> "imagestoreclone:1_monitor_20000 node2" [ style = bold]
"imagestoreclone:1_start_0 node2" -> "imagestorecloneset_running_0" [ style = bold]
"imagestoreclone:1_start_0 node2" [ style=bold color="green" fontcolor="black"]
"imagestorecloneset_confirmed-post_notify_running_0" -> "imagestoreclone:0_monitor_20000 node1" [ style = bold]
"imagestorecloneset_confirmed-post_notify_running_0" -> "imagestoreclone:1_monitor_20000 node2" [ style = bold]
"imagestoreclone:1_monitor_0 node1" -> "imagestorecloneset_start_0" [ style = bold]
"imagestoreclone:1_monitor_0 node1" -> "imagestorecloneset_stopped_0" [ style = bold]
"imagestoreclone:1_monitor_0 node1" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:1_monitor_20000 node1" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:1_post_notify_start_0 node1" -> "imagestorecloneset_confirmed-post_notify_running_0" [ style = bold]
"imagestoreclone:1_post_notify_start_0 node1" [ style=bold color="green" fontcolor="black"]
"imagestoreclone:1_start_0 node1" -> "imagestoreclone:1_monitor_20000 node1" [ style = bold]
"imagestoreclone:1_start_0 node1" -> "imagestorecloneset_running_0" [ style = bold]
"imagestoreclone:1_start_0 node1" [ style=bold color="green" fontcolor="black"]
"imagestorecloneset_confirmed-post_notify_running_0" -> "imagestoreclone:0_monitor_20000 node2" [ style = bold]
"imagestorecloneset_confirmed-post_notify_running_0" -> "imagestoreclone:1_monitor_20000 node1" [ style = bold]
"imagestorecloneset_confirmed-post_notify_running_0" -> "sles10_migrate_to_0 node2" [ style = bold]
"imagestorecloneset_confirmed-post_notify_running_0" -> "sles10_start_0 node1" [ style = bold]
"imagestorecloneset_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
Expand All @@ -105,8 +105,8 @@
"imagestorecloneset_confirmed-pre_notify_stop_0" -> "imagestorecloneset_post_notify_stopped_0" [ style = bold]
"imagestorecloneset_confirmed-pre_notify_stop_0" -> "imagestorecloneset_stop_0" [ style = bold]
"imagestorecloneset_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"imagestorecloneset_post_notify_running_0" -> "imagestoreclone:0_post_notify_start_0 node1" [ style = bold]
"imagestorecloneset_post_notify_running_0" -> "imagestoreclone:1_post_notify_start_0 node2" [ style = bold]
"imagestorecloneset_post_notify_running_0" -> "imagestoreclone:0_post_notify_start_0 node2" [ style = bold]
"imagestorecloneset_post_notify_running_0" -> "imagestoreclone:1_post_notify_start_0 node1" [ style = bold]
"imagestorecloneset_post_notify_running_0" -> "imagestorecloneset_confirmed-post_notify_running_0" [ style = bold]
"imagestorecloneset_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
"imagestorecloneset_post_notify_stopped_0" -> "imagestorecloneset_confirmed-post_notify_stopped_0" [ style = bold]
Expand All @@ -118,8 +118,8 @@
"imagestorecloneset_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"imagestorecloneset_running_0" -> "imagestorecloneset_post_notify_running_0" [ style = bold]
"imagestorecloneset_running_0" [ style=bold color="green" fontcolor="orange"]
"imagestorecloneset_start_0" -> "imagestoreclone:0_start_0 node1" [ style = bold]
"imagestorecloneset_start_0" -> "imagestoreclone:1_start_0 node2" [ style = bold]
"imagestorecloneset_start_0" -> "imagestoreclone:0_start_0 node2" [ style = bold]
"imagestorecloneset_start_0" -> "imagestoreclone:1_start_0 node1" [ style = bold]
"imagestorecloneset_start_0" -> "imagestorecloneset_running_0" [ style = bold]
"imagestorecloneset_start_0" [ style=bold color="green" fontcolor="orange"]
"imagestorecloneset_stop_0" -> "imagestoreclone:0_stop_0 node2" [ style = bold]
Expand Down
34 changes: 10 additions & 24 deletions cts/scheduler/dot/promotable-anon-recover-promoted.dot
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,19 @@
digraph "g" {
"stateful-clone_promote_0" -> "stateful_promote_0 node2" [ style = bold]
"stateful-clone_promote_0" -> "stateful:2_promote_0 node2" [ style = bold]
"stateful-clone_promote_0" [ style=bold color="green" fontcolor="orange"]
"stateful-clone_promoted_0" [ style=bold color="green" fontcolor="orange"]
"stateful-clone_running_0" -> "stateful-clone_promote_0" [ style = bold]
"stateful-clone_running_0" [ style=bold color="green" fontcolor="orange"]
"stateful-clone_start_0" -> "stateful-clone_running_0" [ style = bold]
"stateful-clone_start_0" -> "stateful:2_start_0 node1" [ style = bold]
"stateful-clone_start_0" -> "stateful_start_0 node2" [ style = bold]
"stateful-clone_start_0" -> "stateful:2_start_0 node2" [ style = bold]
"stateful-clone_start_0" [ style=bold color="green" fontcolor="orange"]
"stateful-clone_stop_0" -> "stateful-clone_stopped_0" [ style = bold]
"stateful-clone_stop_0" -> "stateful_stop_0 node1" [ style = bold]
"stateful-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"stateful-clone_stopped_0" -> "stateful-clone_promote_0" [ style = bold]
"stateful-clone_stopped_0" -> "stateful-clone_start_0" [ style = bold]
"stateful-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"stateful:2_monitor_11000 node1" [ style=bold color="green" fontcolor="black"]
"stateful:2_start_0 node1" -> "stateful-clone_running_0" [ style = bold]
"stateful:2_start_0 node1" -> "stateful:2_monitor_11000 node1" [ style = bold]
"stateful:2_start_0 node1" [ style=bold color="green" fontcolor="black"]
"stateful_monitor_10000 node2" [ style=bold color="green" fontcolor="black"]
"stateful:2_monitor_10000 node2" [ style=bold color="green" fontcolor="black"]
"stateful:2_promote_0 node2" -> "stateful-clone_promoted_0" [ style = bold]
"stateful:2_promote_0 node2" -> "stateful:2_monitor_10000 node2" [ style = bold]
"stateful:2_promote_0 node2" [ style=bold color="green" fontcolor="black"]
"stateful:2_start_0 node2" -> "stateful-clone_running_0" [ style = bold]
"stateful:2_start_0 node2" -> "stateful:2_monitor_10000 node2" [ style = bold]
"stateful:2_start_0 node2" -> "stateful:2_promote_0 node2" [ style = bold]
"stateful:2_start_0 node2" [ style=bold color="green" fontcolor="black"]
"stateful_monitor_11000 node3-rem" [ style=bold color="green" fontcolor="black"]
"stateful_promote_0 node2" -> "stateful-clone_promoted_0" [ style = bold]
"stateful_promote_0 node2" -> "stateful_monitor_10000 node2" [ style = bold]
"stateful_promote_0 node2" [ style=bold color="green" fontcolor="black"]
"stateful_start_0 node2" -> "stateful-clone_running_0" [ style = bold]
"stateful_start_0 node2" -> "stateful_monitor_10000 node2" [ style = bold]
"stateful_start_0 node2" -> "stateful_promote_0 node2" [ style = bold]
"stateful_start_0 node2" [ style=bold color="green" fontcolor="black"]
"stateful_stop_0 node1" -> "stateful-clone_stopped_0" [ style = bold]
"stateful_stop_0 node1" -> "stateful_start_0 node2" [ style = bold]
"stateful_stop_0 node1" [ style=bold color="green" fontcolor="black"]
}

0 comments on commit f10da99

Please sign in to comment.