Merge pull request #815 from kgaillot/premote

pacemaker_remote enhancements

attrd/commands.c

@@ -765,10 +765,15 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da
 {
     if ((kind == crm_status_nstate) || (kind == crm_status_rstate)) {
         if (safe_str_eq(peer->state, CRM_NODE_MEMBER)) {
-            if ((election_state(writer) == election_won)) {
+            /* If we're the writer, send new peers a list of all attributes
+             * (unless it's a remote node, which doesn't run its own attrd)
+             */
+            if ((election_state(writer) == election_won)
+                && !is_set(peer->flags, crm_remote_node)) {
                 attrd_peer_sync(peer, NULL);
             }
         } else {
+            /* Remove all attribute values associated with lost nodes */
             attrd_peer_remove(peer->id, peer->uname, FALSE, __FUNCTION__);
             if (peer_writer && safe_str_eq(peer->uname, peer_writer)) {
                 free(peer_writer);

crmd/callbacks.c

@@ -109,7 +109,14 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d
     bool appeared = FALSE;
     const char *status = NULL;
 
-    set_bit(fsa_input_register, R_PEER_DATA);
+    /* Crmd waits to receive some information from the membership layer before
+     * declaring itself operational. If this is being called for a cluster node,
+     * indicate that we have it.
+     */
+    if (!is_set(node->flags, crm_remote_node)) {
+        set_bit(fsa_input_register, R_PEER_DATA);
+    }
+
     if (node->uname == NULL) {
         return;
     }
@@ -132,20 +139,9 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d
                 return;
 
             } else if(safe_str_eq(CRM_NODE_MEMBER, node->state)) {
-                GListPtr gIter = stonith_cleanup_list;
-
                 appeared = TRUE;
-
-                while (gIter != NULL) {
-                    GListPtr tmp = gIter;
-                    char *target = tmp->data;
-
-                    gIter = gIter->next;
-                    if(safe_str_eq(node->uname, target)) {
-                        crm_trace("Removing %s from the cleanup list", target);
-                        stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp);
-                        free(target);
-                    }
+                if (!is_set(node->flags, crm_remote_node)) {
+                    remove_stonith_cleanup(node->uname);
                 }
             }
 

crmd/cib.c

@@ -136,8 +136,7 @@ do_cib_replaced(const char *event, xmlNode * msg)
     }
 
     /* start the join process again so we get everyone's LRM status */
-    populate_cib_nodes(node_update_quick | node_update_cluster | node_update_peer | node_update_join
-                       | node_update_expected, __FUNCTION__);
+    populate_cib_nodes(node_update_quick|node_update_all, __FUNCTION__);
     register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
 }
 

crmd/crmd_lrm.h

@@ -73,8 +73,6 @@ struct pending_deletion_op_s {
     ha_msg_input_t *input;
 };
 
-xmlNode *do_lrm_query_internal(lrm_state_t * lrm_state, gboolean is_replace);
-
 /*!
  * \brief Is this the local ipc connection to the lrmd
  */

crmd/crmd_utils.h

@@ -67,17 +67,14 @@ enum node_update_flags {
     node_update_peer = 0x0020,
     node_update_join = 0x0040,
     node_update_expected = 0x0100,
+    node_update_all = node_update_cluster|node_update_peer|node_update_join|node_update_expected,
 };
 
 gboolean crm_timer_stop(fsa_timer_t * timer);
 gboolean crm_timer_start(fsa_timer_t * timer);
 gboolean crm_timer_popped(gpointer data);
 gboolean is_timer_started(fsa_timer_t * timer);
 
-xmlNode *create_node_state(const char *uname, const char *in_cluster,
-                           const char *is_peer, const char *join_state,
-                           const char *exp_state, gboolean clear_shutdown, const char *src);
-
 int crmd_exit(int rc);
 int crmd_fast_exit(int rc);
 gboolean stop_subsystem(struct crm_subsystem_s *centry, gboolean force_quit);

crmd/election.c

@@ -172,24 +172,12 @@ do_dc_takeover(long long action,
 {
     int rc = pcmk_ok;
     xmlNode *cib = NULL;
-    GListPtr gIter = NULL;
     const char *cluster_type = name_for_cluster_type(get_cluster_type());
     const char *watchdog = NULL;
 
     crm_info("Taking over DC status for this partition");
     set_bit(fsa_input_register, R_THE_DC);
-
-    for (gIter = stonith_cleanup_list; gIter != NULL; gIter = gIter->next) {
-        char *target = gIter->data;
-        crm_node_t *target_node = crm_get_peer(0, target);
-        const char *uuid = crm_peer_uuid(target_node);
-
-        crm_notice("Marking %s, target of a previous stonith action, as clean", target);
-        send_stonith_update(NULL, target, uuid);
-        free(target);
-    }
-    g_list_free(stonith_cleanup_list);
-    stonith_cleanup_list = NULL;
+    execute_stonith_cleanup();
 
 #if SUPPORT_COROSYNC
     if (is_classic_ais_cluster()) {

crmd/fsa.c

@@ -540,8 +540,7 @@ do_state_transition(long long actions,
     }
 
     if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
-        populate_cib_nodes(node_update_quick | node_update_cluster | node_update_peer |
-                           node_update_join | node_update_expected, __FUNCTION__);
+        populate_cib_nodes(node_update_quick|node_update_all, __FUNCTION__);
     }
 
     switch (next_state) {
@@ -554,18 +553,8 @@ do_state_transition(long long actions,
             break;
         case S_NOT_DC:
             election_trigger->counter = 0;
-            if (stonith_cleanup_list) {
-                GListPtr gIter = NULL;
+            purge_stonith_cleanup();
 
-                for (gIter = stonith_cleanup_list; gIter != NULL; gIter = gIter->next) {
-                    char *target = gIter->data;
-
-                    crm_info("Purging %s from stonith cleanup list", target);
-                    free(target);
-                }
-                g_list_free(stonith_cleanup_list);
-                stonith_cleanup_list = NULL;
-            }
             if (is_set(fsa_input_register, R_SHUTDOWN)) {
                 crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
                 set_bit(tmp, A_SHUTDOWN_REQ);

crmd/join_dc.c

@@ -49,6 +49,11 @@ crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase
         return;
     }
 
+    /* Remote nodes do not participate in joins */
+    if (is_set(node->flags, crm_remote_node)) {
+        return;
+    }
+
     last = node->join;
 
     if(phase == last) {

crmd/lrm.c

@@ -831,27 +831,21 @@ build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
     return FALSE;
 }
 
-xmlNode *
-do_lrm_query_internal(lrm_state_t * lrm_state, gboolean is_replace)
+static xmlNode *
+do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags)
 {
     xmlNode *xml_state = NULL;
     xmlNode *xml_data = NULL;
     xmlNode *rsc_list = NULL;
     const char *uuid = NULL;
 
-    if (safe_str_eq(lrm_state->node_name, fsa_our_uname)) {
+    if (lrm_state_is_local(lrm_state)) {
         crm_node_t *peer = crm_get_peer(0, lrm_state->node_name);
-        xml_state = do_update_node_cib(peer, node_update_cluster|node_update_peer, NULL, __FUNCTION__);
-        /* The next two lines shouldn't be necessary for newer DCs */
-        crm_xml_add(xml_state, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
-        crm_xml_add(xml_state, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
+        xml_state = do_update_node_cib(peer, update_flags, NULL, __FUNCTION__);
         uuid = fsa_our_uuid;
 
     } else {
-        xml_state = create_xml_node(NULL, XML_CIB_TAG_STATE);
-        crm_xml_add(xml_state, XML_NODE_IS_REMOTE, "true");
-        crm_xml_add(xml_state, XML_ATTR_ID, lrm_state->node_name);
-        crm_xml_add(xml_state, XML_ATTR_UNAME, lrm_state->node_name);
+        xml_state = simple_remote_node_status(lrm_state->node_name, NULL, __FUNCTION__);
         uuid = lrm_state->node_name;
     }
 
@@ -871,12 +865,23 @@ xmlNode *
 do_lrm_query(gboolean is_replace, const char *node_name)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
+    xmlNode *xml_state;
 
     if (!lrm_state) {
         crm_err("Could not query lrm state for lrmd node %s", node_name);
         return NULL;
     }
-    return do_lrm_query_internal(lrm_state, is_replace);
+    xml_state = do_lrm_query_internal(lrm_state,
+                                      node_update_cluster|node_update_peer);
+
+    /* In case this function is called to generate a join confirmation to
+     * send to the DC, force the current and expected join state to member.
+     * This isn't necessary for newer DCs but is backward compatible.
+     */
+    crm_xml_add(xml_state, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
+    crm_xml_add(xml_state, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
+
+    return xml_state;
 }
 
 static void
@@ -1541,7 +1546,7 @@ do_lrm_invoke(long long action,
 
     if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
         int rc = pcmk_ok;
-        xmlNode *fragment = do_lrm_query_internal(lrm_state, TRUE);
+        xmlNode *fragment = do_lrm_query_internal(lrm_state, node_update_all);
 
         fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
         crm_info("Forced a local LRM refresh: call=%d", rc);
@@ -1562,7 +1567,7 @@ do_lrm_invoke(long long action,
         free_xml(fragment);
 
     } else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
-        xmlNode *data = do_lrm_query_internal(lrm_state, FALSE);
+        xmlNode *data = do_lrm_query_internal(lrm_state, node_update_all);
         xmlNode *reply = create_reply(input->msg, data);
 
         if (relay_message(reply, TRUE) == FALSE) {

crmd/te_actions.c

@@ -82,11 +82,7 @@ send_stonith_update(crm_action_t * action, const char *target, const char *uuid)
     }
 
     crmd_peer_down(peer, TRUE);
-    node_state =
-        do_update_node_cib(peer,
-                           node_update_cluster | node_update_peer | node_update_join |
-                           node_update_expected, NULL, __FUNCTION__);
-
+    node_state = do_update_node_cib(peer, node_update_all, NULL, __FUNCTION__);
 
     /* we have to mark whether or not remote nodes have already been fenced */
     if (peer->flags & crm_remote_node) {

crmd/te_utils.c

@@ -31,7 +31,96 @@
 #include <crm/fencing/internal.h>
 
 crm_trigger_t *stonith_reconnect = NULL;
-GListPtr stonith_cleanup_list = NULL;
+
+/*
+ * stonith cleanup list
+ *
+ * If the DC is shot, proper notifications might not go out.
+ * The stonith cleanup list allows the cluster to (re-)send
+ * notifications once a new DC is elected.
+ */
+
+static GListPtr stonith_cleanup_list = NULL;
+
+/*!
+ * \internal
+ * \brief Add a node to the stonith cleanup list
+ *
+ * \param[in] target  Name of node to add
+ */
+void
+add_stonith_cleanup(const char *target) {
+    stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target));
+}
+
+/*!
+ * \internal
+ * \brief Remove a node from the stonith cleanup list
+ *
+ * \param[in] Name of node to remove
+ */
+void
+remove_stonith_cleanup(const char *target)
+{
+    GListPtr iter = stonith_cleanup_list;
+
+    while (iter != NULL) {
+        GListPtr tmp = iter;
+        char *iter_name = tmp->data;
+
+        iter = iter->next;
+        if (safe_str_eq(target, iter_name)) {
+            crm_trace("Removing %s from the cleanup list", iter_name);
+            stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp);
+            free(iter_name);
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Purge all entries from the stonith cleanup list
+ */
+void
+purge_stonith_cleanup()
+{
+    if (stonith_cleanup_list) {
+        GListPtr iter = NULL;
+
+        for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
+            char *target = iter->data;
+
+            crm_info("Purging %s from stonith cleanup list", target);
+            free(target);
+        }
+        g_list_free(stonith_cleanup_list);
+        stonith_cleanup_list = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Send stonith updates for all entries in cleanup list, then purge it
+ */
+void
+execute_stonith_cleanup()
+{
+    GListPtr iter;
+
+    for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
+        char *target = iter->data;
+        crm_node_t *target_node = crm_get_peer(0, target);
+        const char *uuid = crm_peer_uuid(target_node);
+
+        crm_notice("Marking %s, target of a previous stonith action, as clean", target);
+        send_stonith_update(NULL, target, uuid);
+        free(target);
+    }
+    g_list_free(stonith_cleanup_list);
+    stonith_cleanup_list = NULL;
+}
+
+/* end stonith cleanup list functions */
 
 static gboolean
 fail_incompletable_stonith(crm_graph_t * graph)
@@ -251,7 +340,9 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event)
             }
 
             /* Assume it was our leader if we dont currently have one */
-        } else if (fsa_our_dc == NULL || safe_str_eq(fsa_our_dc, st_event->target)) {
+        } else if (((fsa_our_dc == NULL) || safe_str_eq(fsa_our_dc, st_event->target))
+            && !is_set(peer->flags, crm_remote_node)) {
+
             crm_notice("Target %s our leader %s (recorded: %s)",
                        fsa_our_dc ? "was" : "may have been", st_event->target,
                        fsa_our_dc ? fsa_our_dc : "<unset>");
@@ -263,8 +354,7 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event)
             if (we_are_executioner) {
                 send_stonith_update(NULL, st_event->target, uuid);
             }
-            stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(st_event->target));
-
+            add_stonith_cleanup(st_event->target);
         }
 
         crmd_peer_down(peer, TRUE);

crmd/tengine.h

@@ -23,10 +23,15 @@
 #  include <crm/stonith-ng.h>
 #  include <crm/services.h>
 extern stonith_t *stonith_api;
-extern GListPtr stonith_cleanup_list;
 extern void send_stonith_update(crm_action_t * stonith_action, const char *target,
                                 const char *uuid);
 
+/* stonith cleanup list */
+void add_stonith_cleanup(const char *target);
+void remove_stonith_cleanup(const char *target);
+void purge_stonith_cleanup(void);
+void execute_stonith_cleanup(void);
+
 /* tengine */
 extern crm_action_t *match_down_event(int rc, const char *target, const char *filter, bool quiet);
 extern crm_action_t *get_cancel_action(const char *id, const char *node);