MDEV-26971: JSON file interface to wsrep node state.

Integration with status reporter in wsrep-lib.

Status reporter reports changes in wsrep state and logged errors/
warnings to a json file which then can be read and interpreted by
an external monitoring tool.

Rationale: until the server is fully initialized it is unaccessible
by client and the only source of information is an error log which
is not machine-friendly. Since wsrep node can spend a very long time
in initialization phase (state transfer), it may be a very long time
that automatic tools can't easily monitor its liveness and progression.

New variable: wsrep_status_file specifies the output file name.
If not set, no file is created and no reporting is done.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/include/galera_variables_ok.inc

@@ -1,6 +1,6 @@
 --disable_query_log
 
---let $galera_variables_ok = `SELECT COUNT(*) = 49 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
+--let $galera_variables_ok = `SELECT COUNT(*) = 50 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
 
 --if (!$galera_variables_ok) {
 	--skip Galera number of variables has changed!

mysql-test/include/galera_variables_ok_debug.inc

@@ -1,6 +1,6 @@
 --disable_query_log
 
---let $galera_variables_ok = `SELECT COUNT(*) = 50 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
+--let $galera_variables_ok = `SELECT COUNT(*) = 51 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
 
 --if (!$galera_variables_ok) {
 	--skip Galera number of variables has changed!

mysql-test/suite/galera/r/galera_defaults.result

@@ -3,7 +3,7 @@ connection node_1;
 # Correct Galera library found
 SELECT COUNT(*) `expect 50` FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_%';
 expect 50
-49
+50
 SELECT VARIABLE_NAME, VARIABLE_VALUE
 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
 WHERE VARIABLE_NAME LIKE 'wsrep_%'
@@ -57,6 +57,7 @@ WSREP_SST_AUTH
 WSREP_SST_DONOR	
 WSREP_SST_DONOR_REJECTS_QUERIES	OFF
 WSREP_SST_METHOD	rsync
+WSREP_STATUS_FILE	
 WSREP_SYNC_WAIT	15
 WSREP_TRX_FRAGMENT_SIZE	0
 WSREP_TRX_FRAGMENT_UNIT	bytes

mysql-test/suite/sys_vars/r/sysvars_wsrep.result

@@ -706,6 +706,21 @@ ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 GLOBAL_VALUE_PATH	NULL
+VARIABLE_NAME	WSREP_STATUS_FILE
+SESSION_VALUE	NULL
+GLOBAL_VALUE	
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	VARCHAR
+VARIABLE_COMMENT	wsrep status output filename
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	NULL
+READ_ONLY	YES
+COMMAND_LINE_ARGUMENT	REQUIRED
+GLOBAL_VALUE_PATH	NULL
 VARIABLE_NAME	WSREP_SYNC_WAIT
 SESSION_VALUE	0
 GLOBAL_VALUE	0

mysql-test/suite/wsrep/r/variables.result

@@ -135,6 +135,7 @@ WSREP_SST_DONOR_REJECTS_QUERIES
 WSREP_SST_METHOD
 WSREP_SST_RECEIVE_ADDRESS
 WSREP_START_POSITION
+WSREP_STATUS_FILE
 WSREP_SYNC_WAIT
 WSREP_TRX_FRAGMENT_SIZE
 WSREP_TRX_FRAGMENT_UNIT

mysql-test/suite/wsrep/r/variables_debug.result

@@ -136,6 +136,7 @@ WSREP_SST_DONOR_REJECTS_QUERIES
 WSREP_SST_METHOD
 WSREP_SST_RECEIVE_ADDRESS
 WSREP_START_POSITION
+WSREP_STATUS_FILE
 WSREP_SYNC_WAIT
 WSREP_TRX_FRAGMENT_SIZE
 WSREP_TRX_FRAGMENT_UNIT

sql/CMakeLists.txt

@@ -22,6 +22,7 @@ IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY)
     wsrep_server_service.cc
     wsrep_storage_service.cc
     wsrep_server_state.cc
+    wsrep_status.cc
     wsrep_utils.cc
     wsrep_xid.cc
     wsrep_check_opts.cc

sql/log.cc

@@ -62,6 +62,7 @@
 #include "wsrep_mysqld.h"
 #ifdef WITH_WSREP
 #include "wsrep_trans_observer.h"
+#include "wsrep_status.h"
 #endif /* WITH_WSREP */
 
 #ifdef HAVE_REPLICATION
@@ -9284,6 +9285,16 @@ static void print_buffer_to_file(enum loglevel level, const char *buffer,
 
   fflush(stderr);
 
+#ifdef WITH_WSREP
+  if (level <= WARNING_LEVEL)
+  {
+    wsrep::reporter::log_level const lvl = (level <= ERROR_LEVEL ?
+                                            wsrep::reporter::error :
+                                            wsrep::reporter::warning);
+    Wsrep_status::report_log_msg(lvl, tag, tag_length, buffer, length, skr);
+  }
+#endif /* WITH_WSREP */
+
   mysql_mutex_unlock(&LOCK_error_log);
   DBUG_VOID_RETURN;
 }

sql/sys_vars.cc

@@ -5990,6 +5990,11 @@ static Sys_var_charptr Sys_wsrep_notify_cmd(
        READ_ONLY GLOBAL_VAR(wsrep_notify_cmd), CMD_LINE(REQUIRED_ARG),
        DEFAULT(""));
 
+static Sys_var_charptr_fscs Sys_wsrep_status_file(
+       "wsrep_status_file", "wsrep status output filename",
+       READ_ONLY GLOBAL_VAR(wsrep_status_file), CMD_LINE(REQUIRED_ARG),
+       DEFAULT(""));
+
 static Sys_var_mybool Sys_wsrep_certify_nonPK(
        "wsrep_certify_nonPK", "Certify tables with no primary key",
        GLOBAL_VAR(wsrep_certify_nonPK), 

sql/wsrep_mutex.h

@@ -20,6 +20,7 @@
 #include "wsrep/mutex.hpp"
 
 /* implementation */
+#include "my_global.h"
 #include "my_pthread.h"
 
 class Wsrep_mutex : public wsrep::mutex

sql/wsrep_mysqld.cc

@@ -17,6 +17,7 @@
 #include "sql_plugin.h"                         /* wsrep_plugins_pre_init() */
 #include "my_global.h"
 #include "wsrep_server_state.h"
+#include "wsrep_status.h"
 
 #include "mariadb.h"
 #include <mysqld.h>
@@ -82,6 +83,7 @@ const char *wsrep_start_position;
 const char *wsrep_data_home_dir;
 const char *wsrep_dbug_option;
 const char *wsrep_notify_cmd;
+const char *wsrep_status_file;
 
 ulong   wsrep_debug;                            // Debug level logging
 my_bool wsrep_convert_LOCK_to_trx;              // Convert locking sessions to trx
@@ -763,6 +765,8 @@ int wsrep_init_server()
   wsrep::log::logger_fn(wsrep_log_cb);
   try
   {
+    Wsrep_status::init_once(wsrep_status_file);
+
     std::string server_name;
     std::string server_id;
     std::string node_address;
@@ -831,6 +835,7 @@ void wsrep_deinit_server()
 {
   wsrep_deinit_schema();
   Wsrep_server_state::destroy();
+  Wsrep_status::destroy();
 }
 
 int wsrep_init()

sql/wsrep_mysqld.h

@@ -79,6 +79,7 @@ extern const char* wsrep_start_position;
 extern ulong       wsrep_max_ws_size;
 extern ulong       wsrep_max_ws_rows;
 extern const char* wsrep_notify_cmd;
+extern const char* wsrep_status_file;
 extern my_bool     wsrep_certify_nonPK;
 extern long int    wsrep_protocol_version;
 extern ulong       wsrep_forced_binlog_format;

sql/wsrep_status.cc

@@ -0,0 +1,60 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include "wsrep_status.h"
+
+mysql_mutex_t LOCK_wsrep_status;
+
+#ifdef HAVE_PSI_INTERFACE
+PSI_mutex_key key_LOCK_wsrep_status;
+#endif
+
+Wsrep_mutex*     Wsrep_status::m_mutex    = 0;
+wsrep::reporter* Wsrep_status::m_instance = 0;
+
+void Wsrep_status::report_log_msg(wsrep::reporter::log_level const level,
+                                  const char* const tag, size_t const tag_len,
+                                  const char* const buf, size_t const buf_len,
+                                  double const tstamp)
+{
+  if (!Wsrep_status::m_instance) return;
+
+  Wsrep_status::m_instance->report_log_msg(level,
+    std::string(tag, tag_len) + std::string(buf, buf_len),
+    tstamp);
+}
+
+void Wsrep_status::init_once(const std::string& file_name)
+{
+  if (file_name.length() > 0 && m_instance == 0)
+  {
+    mysql_mutex_init(key_LOCK_wsrep_status, &LOCK_wsrep_status,
+                     MY_MUTEX_INIT_FAST);
+    m_mutex    = new Wsrep_mutex(LOCK_wsrep_status);
+    m_instance = new wsrep::reporter(*m_mutex, file_name, 4);
+  }
+}
+
+void Wsrep_status::destroy()
+{
+  if (m_instance)
+  {
+    delete m_instance;
+    m_instance= 0;
+    delete m_mutex;
+    m_mutex= 0;
+    mysql_mutex_destroy(&LOCK_wsrep_status);
+  }
+}

sql/wsrep_status.h

@@ -0,0 +1,61 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef WSREP_STATUS_H
+#define WSREP_STATUS_H
+
+/* wsrep-lib */
+#include "wsrep/reporter.hpp"
+
+/* implementation */
+#include "wsrep_mutex.h"
+
+class Wsrep_status
+{
+public:
+  static void init_once(const std::string& file_name);
+  static void destroy();
+
+  static void report_state(enum wsrep::server_state::state const state,
+                           float const progress = wsrep::reporter::indefinite)
+  {
+    if (!Wsrep_status::m_instance) return;
+
+    Wsrep_status::m_instance->report_state(state, progress);
+  }
+
+  static void report_log_msg(wsrep::reporter::log_level level,
+                             const char* tag, size_t tag_len,
+                             const char* buf, size_t buf_len,
+                             double const tstamp = wsrep::reporter::undefined);
+//  {
+//    if (!Wsrep_status::m_instance) return;
+//
+//    Wsrep_status::m_instance->report_log_msg(level, msg, tstamp);
+//  }
+
+  static bool is_instance_initialized()
+  {
+      return m_instance;
+  }
+
+private:
+  Wsrep_status(const std::string& file_name);
+
+  static Wsrep_mutex*     m_mutex;
+  static wsrep::reporter* m_instance;
+};
+
+#endif /* WSREP_STATUS_H */

sql/wsrep_utils.cc

@@ -26,6 +26,7 @@
 #include "wsrep_utils.h"
 #include "wsrep_mysqld.h"
 #include "wsrep_thd.h"
+#include "wsrep_status.h"
 
 #include <sql_class.h>
 
@@ -93,6 +94,18 @@ wsrep_prepend_PATH (const char* path)
 namespace wsp
 {
 
+void
+node_status::set(enum wsrep::server_state::state new_status,
+                 const wsrep::view* view)
+{
+    if (status != new_status || 0 != view)
+    {
+        wsrep_notify_status(new_status, view);
+        status= new_status;
+        Wsrep_status::report_state(status, 0);
+    }
+}
+
 bool
 env::ctor_common(char** e)
 {

sql/wsrep_utils.h

@@ -21,6 +21,20 @@
 
 unsigned int wsrep_check_ip (const char* const addr, bool *is_ipv6);
 size_t wsrep_guess_ip (char* buf, size_t buf_len);
+namespace wsp {
+class node_status
+{
+public:
+  node_status() : status(wsrep::server_state::s_disconnected) {}
+  void set(enum wsrep::server_state::state new_status,
+           const wsrep::view* view= 0);
+  enum wsrep::server_state::state get() const { return status; }
+private:
+  enum wsrep::server_state::state status;
+};
+} /* namespace wsp */
+
+extern wsp::node_status local_status;
 
 /* returns the length of the host part of the address string */
 size_t wsrep_host_len(const char* addr, size_t addr_len);