High: crmd: Block after 10 failed fencing attempts for a node

Author: beekhof

TODO.markdown

@@ -1,7 +1,6 @@
 # Semi-random collection of tasks we'd like to get done
 
 ## Targeted for 1.2
-- Block after N stonith failures (per node)
 - Promote any necessary items from the 1.1 schema
 - Get fencing/test.c working again (with and without stonithd -s)
 - Avoid the use of xmlNode in fencing register_callback() call types

crmd/crmd_utils.h

@@ -74,6 +74,7 @@ extern void update_attrd(const char *host, const char *name, const char *value,
                          const char *user_name);
 
 extern const char *get_timer_desc(fsa_timer_t * timer);
+gboolean too_many_st_failures(void);
 
 #  define start_transition(state) do {					\
 	switch(state) {							\

crmd/te_actions.c

@@ -124,9 +124,8 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action)
         return FALSE;
     }
 
-    crm_notice(
-                  "Executing %s fencing operation (%s) on %s (timeout=%d)",
-                  type, id, target, transition_graph->stonith_timeout);
+    crm_notice("Executing %s fencing operation (%s) on %s (timeout=%d)",
+               type, id, target, transition_graph->stonith_timeout);
 
     /* Passing NULL means block until we can connect... */
     te_connect_stonith(NULL);
@@ -138,10 +137,10 @@ te_fence_node(crm_graph_t * graph, crm_action_t * action)
     rc = stonith_api->cmds->fence(stonith_api, options, target, type,
                                   transition_graph->stonith_timeout / 1000);
 
-    stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000,
-                                         FALSE, generate_transition_key(transition_graph->id,
-                                                                        action->id, 0, te_uuid),
-                                         "tengine_stonith_callback", tengine_stonith_callback);
+    stonith_api->cmds->register_callback(
+        stonith_api, rc, transition_graph->stonith_timeout / 1000,
+        FALSE, generate_transition_key(transition_graph->id, action->id, 0, te_uuid),
+        "tengine_stonith_callback", tengine_stonith_callback);
 
     return TRUE;
 }
@@ -481,7 +480,7 @@ notify_crmd(crm_graph_t * graph)
                 if (transition_timer->period_ms > 0) {
                     crm_timer_stop(transition_timer);
                     crm_timer_start(transition_timer);
-                } else {
+                } else if(too_many_st_failures() == FALSE) {
                     event = I_PE_CALC;
                 }
 

crmd/te_callbacks.c

@@ -393,6 +393,34 @@ process_te_message(xmlNode * msg, xmlNode * xml_data)
     return TRUE;
 }
 
+GHashTable *stonith_failures = NULL;
+struct st_fail_rec 
+{
+        int count;
+};
+
+gboolean too_many_st_failures(void) 
+{
+    GHashTableIter iter;
+    const char *key = NULL;
+    struct st_fail_rec *value = NULL;
+
+    if(stonith_failures == NULL) {
+        return FALSE;
+    }
+
+    g_hash_table_iter_init(&iter, stonith_failures);
+    while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
+        if(value->count > 10) {
+            crm_notice("Too many failures to fence %s (%d), giving up",
+                       key, value->count);
+            return TRUE;
+        }
+    }
+    return FALSE;
+}
+
+
 void
 tengine_stonith_callback(stonith_t * stonith, const xmlNode * msg, int call_id, int rc,
                          xmlNode * output, void *userdata)
@@ -402,6 +430,7 @@ tengine_stonith_callback(stonith_t * stonith, const xmlNode * msg, int call_id,
     int stonith_id = -1;
     int transition_id = -1;
     crm_action_t *action = NULL;
+    struct st_fail_rec *rec = NULL;
 
     CRM_CHECK(userdata != NULL, return);
     crm_log_xml_trace(output, "StonithOp");
@@ -411,6 +440,7 @@ tengine_stonith_callback(stonith_t * stonith, const xmlNode * msg, int call_id,
     if (AM_I_DC == FALSE) {
         return;
     }
+
     /* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */
     /*       op->call_id, op->optype, op->node_name, op->op_result, */
     /*       (char *)op->node_list, op->private_data); */
@@ -435,6 +465,11 @@ tengine_stonith_callback(stonith_t * stonith, const xmlNode * msg, int call_id,
     }
 
     stop_te_timer(action->timer);
+    if(stonith_failures == NULL) {
+        stonith_failures = g_hash_table_new_full(
+            crm_str_hash, g_str_equal, g_hash_destroy_str, free);
+    }
+
     if (rc == pcmk_ok) {
         const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
         const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
@@ -444,7 +479,11 @@ tengine_stonith_callback(stonith_t * stonith, const xmlNode * msg, int call_id,
             action->confirmed = TRUE;
             send_stonith_update(action, target, uuid);
         }
-
+        rec = g_hash_table_lookup(stonith_failures, target);
+        if(rec) {
+            rec->count = 0;
+        }
+        
     } else {
         const char *target = crm_element_value_const(action->xml, XML_LRM_ATTR_TARGET);
         const char *allow_fail = crm_meta_value(action->params, XML_ATTR_TE_ALLOWFAIL);
@@ -454,6 +493,15 @@ tengine_stonith_callback(stonith_t * stonith, const xmlNode * msg, int call_id,
             crm_notice("Stonith operation %d for %s failed (%s): aborting transition.", call_id, target, pcmk_strerror(rc));
             abort_transition(INFINITY, tg_restart, "Stonith failed", NULL);
         }
+
+        rec = g_hash_table_lookup(stonith_failures, target);
+        if(rec) {
+            rec->count++;
+        } else {
+            rec = malloc(sizeof(struct st_fail_rec));
+            rec->count = 1;
+            g_hash_table_insert(stonith_failures, strdup(target), rec);
+        }
     }
 
     update_graph(transition_graph, action);

crmd/te_utils.c

@@ -386,7 +386,7 @@ abort_transition_graph(int abort_priority, enum transition_action abort_action,
         if (transition_timer->period_ms > 0) {
             crm_timer_stop(transition_timer);
             crm_timer_start(transition_timer);
-        } else {
+        } else if(too_many_st_failures() == FALSE) {
             register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL);
         }
         return;