Merge pull request #709 from gao-yan/cib-legacy-mode

Fix cib legacy mode

cib/callbacks.c

@@ -40,6 +40,8 @@
 #include <notify.h>
 #include "common.h"
 
+static unsigned long cib_local_bcast_num = 0;
+
 typedef struct cib_local_notify_s {
     xmlNode *notify_src;
     char *client_id;
@@ -48,7 +50,13 @@ typedef struct cib_local_notify_s {
 } cib_local_notify_t;
 
 int next_client_id = 0;
+
+#if SUPPORT_PLUGIN
+gboolean legacy_mode = TRUE;
+#else
 gboolean legacy_mode = FALSE;
+#endif
+
 qb_ipcs_service_t *ipcs_ro = NULL;
 qb_ipcs_service_t *ipcs_rw = NULL;
 qb_ipcs_service_t *ipcs_shm = NULL;
@@ -82,8 +90,12 @@ static gboolean cib_read_legacy_mode(void)
     return legacy;
 }
 
-static gboolean cib_legacy_mode(void)
+gboolean cib_legacy_mode(void)
 {
+#if SUPPORT_PLUGIN
+    return TRUE;
+#endif
+
     if(cib_read_legacy_mode()) {
         return TRUE;
     }
@@ -441,6 +453,54 @@ do_local_notify(xmlNode * notify_src, const char *client_id,
     }
 }
 
+static void
+local_notify_destroy_callback(gpointer data)
+{
+    cib_local_notify_t *notify = data;
+
+    free_xml(notify->notify_src);
+    free(notify->client_id);
+    free(notify);
+}
+
+static void
+check_local_notify(int bcast_id)
+{
+    cib_local_notify_t *notify = NULL;
+
+    if (!local_notify_queue) {
+        return;
+    }
+
+    notify = g_hash_table_lookup(local_notify_queue, GINT_TO_POINTER(bcast_id));
+
+    if (notify) {
+        do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
+                        notify->from_peer);
+        g_hash_table_remove(local_notify_queue, GINT_TO_POINTER(bcast_id));
+    }
+}
+
+static void
+queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
+                   gboolean from_peer)
+{
+    cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));
+
+    notify->notify_src = notify_src;
+    notify->client_id = strdup(client_id);
+    notify->sync_reply = sync_reply;
+    notify->from_peer = from_peer;
+
+    if (!local_notify_queue) {
+        local_notify_queue = g_hash_table_new_full(g_direct_hash,
+                                                   g_direct_equal, NULL,
+                                                   local_notify_destroy_callback);
+    }
+
+    g_hash_table_insert(local_notify_queue, GINT_TO_POINTER(cib_local_bcast_num), notify);
+}
+
 static void
 parse_local_options_v1(crm_client_t * cib_client, int call_type, int call_options, const char *host,
                     const char *op, gboolean * local_notify, gboolean * needs_reply,
@@ -1039,6 +1099,27 @@ cib_process_request(xmlNode * request, gboolean force_synchronous, gboolean priv
          */
         crm_trace("Completed slave update");
 
+    } else if (cib_legacy_mode() &&
+               rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
+        gboolean broadcast = FALSE;
+
+        cib_local_bcast_num++;
+        crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
+        broadcast = send_peer_reply(request, result_diff, originator, TRUE);
+
+        if (broadcast && client_id && local_notify && op_reply) {
+
+            /* If we have been asked to sync the reply,
+             * and a bcast msg has gone out, we queue the local notify
+             * until we know the bcast message has been received */
+            local_notify = FALSE;
+            crm_trace("Queuing local %ssync notification for %s",
+                      (call_options & cib_sync_call) ? "" : "a-", client_id);
+
+            queue_local_notify(op_reply, client_id, (call_options & cib_sync_call), from_peer);
+            op_reply = NULL;    /* the reply is queued, so don't free here */
+        }
+
     } else if (call_options & cib_discard_reply) {
         crm_trace("Caller isn't interested in reply");
 
@@ -1115,6 +1196,10 @@ cib_process_command(xmlNode * request, xmlNode ** reply, xmlNode ** cib_diff, gb
     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
     rc = cib_get_operation_id(op, &call_type);
 
+    if (cib_legacy_mode()) {
+        call_options |= cib_force_digest;
+    }
+
     if (rc == pcmk_ok && privileged == FALSE) {
         rc = cib_op_can_run(call_type, call_options, privileged, global_update);
     }
@@ -1322,6 +1407,11 @@ cib_peer_callback(xmlNode * msg, void *private_data)
 
     if (cib_legacy_mode() && (originator == NULL || crm_str_eq(originator, cib_our_uname, TRUE))) {
         /* message is from ourselves */
+        int bcast_id = 0;
+
+        if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
+            check_local_notify(bcast_id);
+        }
         return;
 
     } else if (crm_peer_cache == NULL) {

cib/callbacks.h

@@ -73,6 +73,8 @@ void cib_shutdown(int nsig);
 void initiate_exit(void);
 void terminate_cib(const char *caller, gboolean fast);
 
+extern gboolean cib_legacy_mode(void);
+
 #if SUPPORT_HEARTBEAT
 extern void cib_ha_peer_callback(HA_Message * msg, void *private_data);
 extern int cib_ccm_dispatch(gpointer user_data);

cib/messages.c

@@ -297,7 +297,14 @@ cib_process_upgrade_server(const char *op, int options, const char *section, xml
             crm_xml_add(up, F_CIB_CALLOPTS, crm_element_value(req, F_CIB_CALLOPTS));
             crm_xml_add(up, F_CIB_CALLID, crm_element_value(req, F_CIB_CALLID));
 
-            send_cluster_message(NULL, crm_msg_cib, up, FALSE);
+            if (cib_legacy_mode() && cib_is_master) {
+                rc = cib_process_upgrade(
+                    op, options, section, up, input, existing_cib, result_cib, answer);
+
+            } else {
+                send_cluster_message(NULL, crm_msg_cib, up, FALSE);
+            }
+
             free_xml(up);
 
         } else if(rc == pcmk_ok) {

include/crm/cib.h

@@ -72,7 +72,8 @@ enum cib_call_options {
 	cib_inhibit_notify  = 0x00010000,
  	cib_quorum_override = 0x00100000,
 	cib_inhibit_bcast   = 0x01000000, /* TODO: Remove */
-	cib_force_diff	    = 0x10000000
+	cib_force_diff	    = 0x10000000,
+	cib_force_digest    = 0x20000000
 };
 
 #define cib_default_options = cib_none

lib/cib/cib_utils.c

@@ -302,6 +302,7 @@ cib_perform_op(const char *op, int call_options, cib_op_t * fn, gboolean is_quer
     const char *new_version = NULL;
     static struct qb_log_callsite *diff_cs = NULL;
     const char *user = crm_element_value(req, F_CIB_USER);
+    bool with_digest = FALSE;
 
     crm_trace("Begin %s%s op", is_query ? "read-only " : "", op);
 
@@ -443,18 +444,21 @@ cib_perform_op(const char *op, int call_options, cib_op_t * fn, gboolean is_quer
     strip_text_nodes(scratch);
     fix_plus_plus_recursive(scratch);
 
+    if (is_set(call_options, cib_force_digest)) {
+        with_digest = TRUE;
+    }
+
     if (is_set(call_options, cib_zero_copy)) {
         /* At this point, current_cib is just the 'cib' tag and its properties,
          *
          * The v1 format would barf on this, but we know the v2 patch
          * format only needs it for the top-level version fields
          */
-        local_diff = xml_create_patchset(2, current_cib, scratch, (bool*)config_changed, manage_counters, FALSE);
+        local_diff = xml_create_patchset(2, current_cib, scratch, (bool*)config_changed, manage_counters, with_digest);
 
     } else {
         static time_t expires = 0;
         time_t tm_now = time(NULL);
-        bool with_digest = FALSE;
 
         if (expires < tm_now) {
             expires = tm_now + 60;  /* Validate clients are correctly applying v2-style diffs at most once a minute */

lib/common/xml.c

@@ -3408,12 +3408,18 @@ dump_xml_attr(xmlAttrPtr attr, int options, char **buffer, int *offset, int *max
 {
     char *p_value = NULL;
     const char *p_name = NULL;
+    xml_private_t *p = NULL;
 
     CRM_ASSERT(buffer != NULL);
     if (attr == NULL || attr->children == NULL) {
         return;
     }
 
+    p = attr->_private;
+    if (is_set(p->flags, xpf_deleted)) {
+        return;
+    }
+
     p_name = (const char *)attr->name;
     p_value = crm_xml_escape((const char *)attr->children->content);
     buffer_print(*buffer, *max, *offset, " %s=\"%s\"", p_name, p_value);