Skip to content

Commit

Permalink
Feature: scheduler, fencer: Delay fencing of lost significant nodes w…
Browse files Browse the repository at this point in the history 
…ith `priority-fencing-delay`
  • Loading branch information
@gao-yan
gao-yan committed Feb 13, 2020
1 parent df6c286 commit e55e71b
Show file tree
Hide file tree
Showing 19 changed files with 206 additions and 101 deletions.
8 changes: 5 additions & 3 deletions daemons/controld/controld_fencing.c
View file
Expand Up @@ -813,13 +813,15 @@ te_fence_node(crm_graph_t *graph, crm_action_t *action)
const char *uuid = NULL;
const char *target = NULL;
const char *type = NULL;
const char *priority_delay = NULL;
gboolean invalid_action = FALSE;
enum stonith_call_options options = st_opt_none;

id = ID(action->xml);
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
type = crm_meta_value(action->params, "stonith_action");
priority_delay = crm_meta_value(action->params, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);

CRM_CHECK(id != NULL, invalid_action = TRUE);
CRM_CHECK(uuid != NULL, invalid_action = TRUE);
Expand All @@ -832,8 +834,8 @@ te_fence_node(crm_graph_t *graph, crm_action_t *action)
}

crm_notice("Requesting fencing (%s) of node %s "
CRM_XS " action=%s timeout=%u",
type, target, id, transition_graph->stonith_timeout);
CRM_XS " action=%s timeout=%u priority_delay=%d",
type, target, id, transition_graph->stonith_timeout, priority_delay);

/* Passing NULL means block until we can connect... */
te_connect_stonith(NULL);
Expand All @@ -844,7 +846,7 @@ te_fence_node(crm_graph_t *graph, crm_action_t *action)

rc = stonith_api->cmds->fence(stonith_api, options, target, type,
(int) (transition_graph->stonith_timeout / 1000),
0);
0, crm_atoi(priority_delay, "-1"));

stonith_api->cmds->register_callback(stonith_api, rc,
(int) (transition_graph->stonith_timeout / 1000),
Expand Down
30 changes: 15 additions & 15 deletions daemons/fenced/cts-fence-helper.c
View file
Expand Up @@ -175,10 +175,10 @@ run_fence_failure_test(void)
cmds->register_device(st, st_opts, "test-id1", "stonith-ng", "fence_dummy", params),
"Register device1 for failure test", 1, 0);

single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0),
single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0, -1),
"Fence failure results off", 1, -pcmk_err_generic);

single_test(st->cmds->fence(st, st_opts, "false_1_node2", "reboot", 3, 0),
single_test(st->cmds->fence(st, st_opts, "false_1_node2", "reboot", 3, 0, -1),
"Fence failure results reboot", 1, -pcmk_err_generic);

single_test(st->cmds->remove_device(st, st_opts, "test-id1"),
Expand Down Expand Up @@ -207,11 +207,11 @@ run_fence_failure_rollover_test(void)
cmds->register_device(st, st_opts, "test-id2", "stonith-ng", "fence_dummy", params),
"Register device2 for rollover test", 1, 0);

single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0),
single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0, -1),
"Fence rollover results off", 1, 0);

/* Expect -ENODEV because fence_dummy requires 'on' to be executed on target */
single_test(st->cmds->fence(st, st_opts, "false_1_node2", "on", 3, 0),
single_test(st->cmds->fence(st, st_opts, "false_1_node2", "on", 3, 0, -1),
"Fence rollover results on", 1, -ENODEV);

single_test(st->cmds->remove_device(st, st_opts, "test-id1"),
Expand Down Expand Up @@ -248,14 +248,14 @@ run_standard_test(void)
single_test(st->cmds->status(st, st_opts, "test-id", "false_1_node1", 1),
"Status false_1_node1", 1, 0);

single_test(st->cmds->fence(st, st_opts, "unknown-host", "off", 1, 0),
single_test(st->cmds->fence(st, st_opts, "unknown-host", "off", 1, 0, -1),
"Fence unknown-host (expected failure)", 0, -ENODEV);

single_test(st->cmds->fence(st, st_opts, "false_1_node1", "off", 1, 0),
single_test(st->cmds->fence(st, st_opts, "false_1_node1", "off", 1, 0, -1),
"Fence false_1_node1", 1, 0);

/* Expect -ENODEV because fence_dummy requires 'on' to be executed on target */
single_test(st->cmds->fence(st, st_opts, "false_1_node1", "on", 1, 0),
single_test(st->cmds->fence(st, st_opts, "false_1_node1", "on", 1, 0, -1),
"Unfence false_1_node1", 1, -ENODEV);

/* Confirm that an invalid level index is rejected */
Expand Down Expand Up @@ -321,31 +321,31 @@ standard_dev_test(void)
rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
crm_debug("Status false_1_node1: %d", rc);

rc = st->cmds->fence(st, st_opts, "unknown-host", "off", 60, 0);
rc = st->cmds->fence(st, st_opts, "unknown-host", "off", 60, 0, -1);
crm_debug("Fence unknown-host: %d", rc);

rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
crm_debug("Status false_1_node1: %d", rc);

rc = st->cmds->fence(st, st_opts, "false_1_node1", "off", 60, 0);
rc = st->cmds->fence(st, st_opts, "false_1_node1", "off", 60, 0, -1);
crm_debug("Fence false_1_node1: %d", rc);

rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
crm_debug("Status false_1_node1: %d", rc);

rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0);
rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0, -1);
crm_debug("Unfence false_1_node1: %d", rc);

rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
crm_debug("Status false_1_node1: %d", rc);

rc = st->cmds->fence(st, st_opts, "some-host", "off", 10, 0);
rc = st->cmds->fence(st, st_opts, "some-host", "off", 10, 0, -1);
crm_debug("Fence alias: %d", rc);

rc = st->cmds->status(st, st_opts, "test-id", "some-host", 10);
crm_debug("Status alias: %d", rc);

rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0);
rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0, -1);
crm_debug("Unfence false_1_node1: %d", rc);

rc = st->cmds->remove_device(st, st_opts, "test-id");
Expand Down Expand Up @@ -387,7 +387,7 @@ test_async_fence_pass(int check_event)
return;
}

rc = st->cmds->fence(st, 0, "true_1_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
rc = st->cmds->fence(st, 0, "true_1_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0, -1);
if (rc < 0) {
crm_err("fence failed with rc %d", rc);
mainloop_test_done(FALSE);
Expand Down Expand Up @@ -420,7 +420,7 @@ test_async_fence_custom_timeout(int check_event)
}
begin = time(NULL);

rc = st->cmds->fence(st, 0, "custom_timeout_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
rc = st->cmds->fence(st, 0, "custom_timeout_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0, -1);
if (rc < 0) {
crm_err("fence failed with rc %d", rc);
mainloop_test_done(FALSE);
Expand All @@ -443,7 +443,7 @@ test_async_fence_timeout(int check_event)
return;
}

rc = st->cmds->fence(st, 0, "false_1_node2", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
rc = st->cmds->fence(st, 0, "false_1_node2", "off", MAINLOOP_DEFAULT_TIMEOUT, 0, -1);
if (rc < 0) {
crm_err("fence failed with rc %d", rc);
mainloop_test_done(FALSE);
Expand Down
86 changes: 51 additions & 35 deletions daemons/fenced/fenced_commands.c
View file
Expand Up @@ -46,6 +46,8 @@ struct device_search_s {
char *action;
/* timeout to use if a device is queried dynamically for possible targets */
int per_device_timeout;
/* enforced fencine delay */
int delay;
/* number of registered fencing devices at time of request */
int replies_needed;
/* number of device replies received so far */
Expand Down Expand Up @@ -78,7 +80,7 @@ typedef struct async_command_s {
int default_timeout; /* seconds */
int timeout; /* seconds */

int start_delay; /* milliseconds */
int start_delay; /* seconds */
int delay_id;

char *op;
Expand Down Expand Up @@ -121,36 +123,36 @@ static int
get_action_delay_max(stonith_device_t * device, const char * action)
{
const char *value = NULL;
int delay_max_ms = 0;
int delay_max = 0;

if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) {
return 0;
}

value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_MAX);
if (value) {
delay_max_ms = crm_get_msec(value);
delay_max = crm_parse_interval_spec(value) / 1000;
}

return delay_max_ms;
return delay_max;
}

static int
get_action_delay_base(stonith_device_t * device, const char * action)
{
const char *value = NULL;
int delay_base_ms = 0;
int delay_base = 0;

if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) {
return 0;
}

value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_BASE);
if (value) {
delay_base_ms = crm_get_msec(value);
delay_base = crm_parse_interval_spec(value) / 1000;
}

return delay_base_ms;
return delay_base;
}

/*!
Expand Down Expand Up @@ -347,7 +349,7 @@ stonith_device_execute(stonith_device_t * device)

if (pending_op && pending_op->delay_id) {
crm_trace
("Operation '%s'%s%s on %s was asked to run too early, waiting for start_delay timeout of %dms",
("Operation '%s'%s%s on %s was asked to run too early, waiting for start_delay timeout of %ds",
pending_op->action, pending_op->victim ? " targeting " : "",
pending_op->victim ? pending_op->victim : "",
device->id, pending_op->start_delay);
Expand Down Expand Up @@ -494,30 +496,36 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
device->pending_ops = g_list_append(device->pending_ops, cmd);
mainloop_set_trigger(device->work);

delay_max = get_action_delay_max(device, cmd->action);
delay_base = get_action_delay_base(device, cmd->action);
if (delay_max == 0) {
delay_max = delay_base;
}
if (delay_max < delay_base) {
crm_warn("Base-delay (%dms) is larger than max-delay (%dms) "
"for %s on %s - limiting to max-delay",
delay_base, delay_max, cmd->action, device->id);
delay_base = delay_max;
// No enforced fencing delay
if (cmd->start_delay < 0 ) {
delay_max = get_action_delay_max(device, cmd->action);
delay_base = get_action_delay_base(device, cmd->action);
if (delay_max == 0) {
delay_max = delay_base;
}
if (delay_max < delay_base) {
crm_warn("Base-delay (%ds) is larger than max-delay (%ds) "
"for %s on %s - limiting to max-delay",
delay_base, delay_max, cmd->action, device->id);
delay_base = delay_max;
}
if (delay_max > 0) {
// coverity[dont_call] We're not using rand() for security
cmd->start_delay =
((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
+ delay_base;
}
}
if (delay_max > 0) {
// coverity[dont_call] We're not using rand() for security
cmd->start_delay =
((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
+ delay_base;
crm_notice("Delaying '%s' action%s%s on %s for %dms (timeout=%ds, base=%dms, "
"max=%dms)",
cmd->action,
cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
device->id, cmd->start_delay, cmd->timeout,
delay_base, delay_max);

if (cmd->start_delay > 0) {
crm_notice("Delaying '%s' action%s%s on %s for %ds (timeout=%ds, base=%ds, "
"max=%ds)",
cmd->action,
cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
device->id, cmd->start_delay, cmd->timeout,
delay_base, delay_max);
cmd->delay_id =
g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
}
}

Expand Down Expand Up @@ -938,6 +946,7 @@ schedule_internal_command(const char *origin,
const char *action,
const char *victim,
int timeout,
int delay,
void *internal_user_data,
void (*done_cb) (GPid pid, int rc, const char *output,
gpointer user_data))
Expand All @@ -949,6 +958,7 @@ schedule_internal_command(const char *origin,
cmd->id = -1;
cmd->default_timeout = timeout ? timeout : 60;
cmd->timeout = cmd->default_timeout;
cmd->start_delay = delay;
cmd->action = strdup(action);
cmd->victim = victim ? strdup(victim) : NULL;
cmd->device = strdup(device->id);
Expand Down Expand Up @@ -1640,7 +1650,8 @@ can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *searc
check_type, dev->id, search->host, search->action);

schedule_internal_command(__FUNCTION__, dev, "list", NULL,
search->per_device_timeout, search, dynamic_list_search_cb);
search->per_device_timeout, search->delay,
search, dynamic_list_search_cb);

/* we'll respond to this search request async in the cb */
return;
Expand All @@ -1654,7 +1665,8 @@ can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *searc
crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
check_type, dev->id, search->host, search->action);
schedule_internal_command(__FUNCTION__, dev, "status", search->host,
search->per_device_timeout, search, status_search_cb);
search->per_device_timeout, search->delay,
search, status_search_cb);
/* we'll respond to this search request async in the cb */
return;
} else {
Expand Down Expand Up @@ -1687,7 +1699,8 @@ search_devices(gpointer key, gpointer value, gpointer user_data)

#define DEFAULT_QUERY_TIMEOUT 20
static void
get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data,
get_capable_devices(const char *host, const char *action, int timeout,
int delay, bool suicide, void *user_data,
void (*callback) (GList * devices, void *user_data))
{
struct device_search_s *search;
Expand Down Expand Up @@ -1740,6 +1753,7 @@ get_capable_devices(const char *host, const char *action, int timeout, bool suic
/* We are guaranteed this many replies. Even if a device gets
* unregistered some how during the async search, we will get
* the correct number of replies. */
search->delay = delay;
search->replies_needed = g_hash_table_size(device_list);
search->allow_suicide = suicide;
search->callback = callback;
Expand Down Expand Up @@ -1958,9 +1972,11 @@ stonith_query(xmlNode * msg, const char *remote_peer, const char *client_id, int
const char *action = NULL;
const char *target = NULL;
int timeout = 0;
int delay = -1;
xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_NEVER);

crm_element_value_int(msg, F_STONITH_TIMEOUT, &timeout);
crm_element_value_int(msg, F_STONITH_PRIORITY_DELAY, &delay);
if (dev) {
const char *device = crm_element_value(dev, F_STONITH_DEVICE);

Expand All @@ -1982,7 +1998,7 @@ stonith_query(xmlNode * msg, const char *remote_peer, const char *client_id, int
query->action = action ? strdup(action) : NULL;
query->call_options = call_options;

get_capable_devices(target, action, timeout,
get_capable_devices(target, action, timeout, delay,
is_set(call_options, st_opt_allow_suicide),
query, stonith_query_capable_device_cb);
}
Expand Down Expand Up @@ -2302,7 +2318,7 @@ stonith_fence(xmlNode * msg)
}

/* If we get to here, then self-fencing is implicitly allowed */
get_capable_devices(host, cmd->action, cmd->default_timeout,
get_capable_devices(host, cmd->action, cmd->default_timeout, cmd->start_delay,
TRUE, cmd, stonith_fence_get_devices_cb);
}

Expand Down
1 change: 1 addition & 0 deletions include/crm/fencing/internal.h
View file
Expand Up @@ -68,6 +68,7 @@ stonith_history_t *stonith__sort_history(stonith_history_t *history);
/*! Timeout period per a device execution */
# define F_STONITH_TIMEOUT "st_timeout"
# define F_STONITH_TOLERANCE "st_tolerance"
# define F_STONITH_PRIORITY_DELAY "st_priority_delay"
/*! Action specific timeout period returned in query of fencing devices. */
# define F_STONITH_ACTION_TIMEOUT "st_action_timeout"
/*! Host in query result is not allowed to run this action */
Expand Down
1 change: 1 addition & 0 deletions include/crm/msg_xml.h
View file
Expand Up @@ -348,6 +348,7 @@ extern "C" {
# define XML_CONFIG_ATTR_FENCE_REACTION "fence-reaction"
# define XML_CONFIG_ATTR_SHUTDOWN_LOCK "shutdown-lock"
# define XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT "shutdown-lock-limit"
# define XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY "priority-fencing-delay"

# define XML_ALERT_ATTR_PATH "path"
# define XML_ALERT_ATTR_TIMEOUT "timeout"
Expand Down
4 changes: 2 additions & 2 deletions include/crm/pengine/internal.h
View file
Expand Up @@ -374,7 +374,7 @@ typedef struct op_digest_cache_s {
op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
pe_working_set_t * data_set);

action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set);
action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, bool priority_delay, pe_working_set_t * data_set);
void trigger_unfencing(
resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set);

Expand All @@ -392,7 +392,7 @@ gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj
void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options,
void * print_data, gboolean print_all);
void pe__rscs_brief_output(pcmk__output_t *out, GListPtr rsc_list, long options, gboolean print_all);
void pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason);
void pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason, bool priority_delay);

node_t *pe_create_node(const char *id, const char *uname, const char *type,
const char *score, pe_working_set_t * data_set);
Expand Down

0 comments on commit e55e71b

Please sign in to comment.