quorum: Add support for nodelist callback

Current quorum callback contains only actual view list and there is no
way how to find out joined/left nodes. This cannot be emulated by user
app, because when corosync restarts before other nodes notices then view
list is unchanged (ring id is changed tho).

Solution is to implement similar callback as for cpg which contains ring
id, member list, joined list and left list.

To implement such callback and keep backwards compatibility,
quorum_model_initialize is introduced. Its behavior is similar to
cpg_model_initialize. This allows passing model v1, which contains
enhanced quorum (full ring id is passed instead of just seq number)
and nodelist callbacks.

To find out which events should be sent by corosync daemon, new message
MESSAGE_REQ_QUORUM_MODEL_GETTYPE is used. Quorum library on init was
sending MESSAGE_REQ_QUORUM_GETTYPE. Whem model v1 is requested the
MESSAGE_REQ_QUORUM_MODEL_GETTYPE is used, which contains model number
so corosync knows that client is using model v1 and can send enhanced
quorum and nodelist events.

Nodelist event is (for now) send both in case of change of membership
and also when requested, also when CS_TRACK_CURRENT is requested, but
then left_list and joined_list is left empty, because they don't make
too much sense there.

New test application testquorummodel is added as an example of new API
usage.

Also during patch developement, I found few bugs here and there, which
are also fixed:
- quorum_initialize was never returning error code returned by
  MESSAGE_REQ_QUORUM_GETTYPE call (always returned CS_OK)
- Allocated memory in send_library_notification was based
  on sizeof(unsigned int) instead of mar_uint32_t. That's not wrong,
  but   it make more sense to use sizeof(mar_uint32_t) instead

(big thanks to Chrissie for englishify the man pages)

Signed-off-by: Jan Friesse <jfriesse@redhat.com>
Reviewed-by: Christine Caulfield <ccaulfie@redhat.com>

include/corosync/ipc_quorum.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008-2011 Red Hat, Inc.
+ * Copyright (c) 2008-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -44,7 +44,8 @@ enum req_quorum_types {
 	MESSAGE_REQ_QUORUM_GETQUORATE = 0,
 	MESSAGE_REQ_QUORUM_TRACKSTART,
 	MESSAGE_REQ_QUORUM_TRACKSTOP,
-	MESSAGE_REQ_QUORUM_GETTYPE
+	MESSAGE_REQ_QUORUM_GETTYPE,
+	MESSAGE_REQ_QUORUM_MODEL_GETTYPE
 };
 
 /**
@@ -55,9 +56,25 @@ enum res_quorum_types {
 	MESSAGE_RES_QUORUM_TRACKSTART,
 	MESSAGE_RES_QUORUM_TRACKSTOP,
 	MESSAGE_RES_QUORUM_NOTIFICATION,
-	MESSAGE_RES_QUORUM_GETTYPE
+	MESSAGE_RES_QUORUM_GETTYPE,
+	MESSAGE_RES_QUORUM_MODEL_GETTYPE,
+	MESSAGE_RES_QUORUM_V1_QUORUM_NOTIFICATION,
+	MESSAGE_RES_QUORUM_V1_NODELIST_NOTIFICATION
 };
 
+/*
+ * Must be in sync with definition in quorum.h
+ */
+enum lib_quorum_model {
+	LIB_QUORUM_MODEL_V0 = 0,
+	LIB_QUORUM_MODEL_V1 = 1,
+};
+
+typedef struct {
+	mar_uint32_t nodeid __attribute__((aligned(8)));
+	mar_uint64_t seq __attribute__((aligned(8)));
+} mar_quorum_ring_id_t;
+
 /**
  * @brief The req_lib_quorum_trackstart struct
  */
@@ -85,6 +102,25 @@ struct res_lib_quorum_notification {
 	mar_uint32_t view_list[];
 };
 
+struct res_lib_quorum_v1_quorum_notification {
+	struct qb_ipc_response_header header __attribute__((aligned(8)));
+	mar_int32_t quorate __attribute__((aligned(8)));
+	mar_quorum_ring_id_t ring_id __attribute__((aligned(8)));
+	mar_uint32_t view_list_entries __attribute__((aligned(8)));
+	mar_uint32_t view_list[];
+};
+
+struct res_lib_quorum_v1_nodelist_notification {
+	struct qb_ipc_response_header header __attribute__((aligned(8)));
+	mar_quorum_ring_id_t ring_id __attribute__((aligned(8)));
+	mar_uint32_t member_list_entries __attribute__((aligned(8)));
+	mar_uint32_t joined_list_entries __attribute__((aligned(8)));
+	mar_uint32_t left_list_entries __attribute__((aligned(8)));
+	mar_uint32_t member_list[];
+//	mar_uint32_t joined_list[];
+//	mar_uint32_t left_list[];
+};
+
 /**
  * @brief The res_lib_quorum_gettype struct
  */
@@ -93,4 +129,14 @@ struct res_lib_quorum_gettype {
 	mar_uint32_t quorum_type;
 };
 
+struct req_lib_quorum_model_gettype {
+        struct qb_ipc_request_header header __attribute__((aligned(8)));
+        mar_uint32_t model __attribute__((aligned(8)));
+};
+
+struct res_lib_quorum_model_gettype {
+	struct qb_ipc_response_header header __attribute__((aligned(8)));
+	mar_uint32_t quorum_type __attribute__((aligned(8)));
+};
+
 #endif

include/corosync/quorum.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008-2012 Red Hat, Inc.
+ * Copyright (c) 2008-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -40,11 +40,21 @@
 extern "C" {
 #endif
 
+typedef enum {
+	QUORUM_MODEL_V0 = 0,
+	QUORUM_MODEL_V1 = 1,
+} quorum_model_t;
+
 /**
  * @brief quorum_handle_t
  */
 typedef uint64_t quorum_handle_t;
 
+struct quorum_ring_id {
+	uint32_t nodeid;
+	uint64_t seq;
+};
+
 /**
  * @brief The quorum_notification_fn_t callback
  */
@@ -56,13 +66,43 @@ typedef void (*quorum_notification_fn_t) (
 	uint32_t *view_list
 	);
 
+typedef void (*quorum_v1_quorum_notification_fn_t) (
+	quorum_handle_t handle,
+	uint32_t quorate,
+	struct quorum_ring_id ring_id,
+	uint32_t member_list_entries, const uint32_t *member_list
+);
+
+typedef void (*quorum_v1_nodelist_notification_fn_t) (
+	quorum_handle_t handle,
+	struct quorum_ring_id ring_id,
+	uint32_t member_list_entries, const uint32_t *member_list,
+	uint32_t joined_list_entries, const uint32_t *joined_list,
+	uint32_t left_list_entries, const uint32_t *left_list
+);
+
 /**
  * @brief The quorum_callbacks_t struct
  */
 typedef struct {
 	quorum_notification_fn_t quorum_notify_fn;
 } quorum_callbacks_t;
 
+typedef struct {
+	quorum_model_t model;
+} quorum_model_data_t;
+
+typedef struct {
+	quorum_model_t model;
+	quorum_notification_fn_t quorum_notify_fn;
+} quorum_model_v0_data_t;
+
+typedef struct {
+	quorum_model_t model;
+	quorum_v1_quorum_notification_fn_t quorum_notify_fn;
+	quorum_v1_nodelist_notification_fn_t nodelist_notify_fn;
+} quorum_model_v1_data_t;
+
 #define QUORUM_FREE	0
 #define QUORUM_SET	1
 
@@ -78,6 +118,13 @@ cs_error_t quorum_initialize (
 	quorum_callbacks_t *callbacks,
 	uint32_t *quorum_type);
 
+cs_error_t quorum_model_initialize (
+	quorum_handle_t *handle,
+	quorum_model_t model,
+	quorum_model_data_t *model_data,
+	uint32_t *quorum_type,
+	void *context);
+
 /**
  * @brief Close the quorum handle
  * @param handle

lib/Makefile.am

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009 Red Hat, Inc.
+# Copyright (c) 2009-2020 Red Hat, Inc.
 #
 # Authors: Andrew Beekhof
 #	   Steven Dake (sdake@redhat.com)

lib/libquorum.verso

@@ -1 +1 @@
-5.0.0
+5.1.0