WL#8688: Support ability to persist SET GLOBAL settings

This WL introduces an option to persist global dynamic configuration variables,
for example like SET PERSIST innodb_flush_log_at_timeout= 14;. Configuration
variables over a connection are lost after server restart. This WL provides
DBAs a way to store configuration variables in a persistent way and allow
server to read and apply all those variables which are persisted during a
restart. A new config file name mysqld-auto.cnf will be created in datadir
when a variable is persisted. This new config file is in JSON format.

In addition to the persistence we add a performance schema table called
"variables_info" which will have an entry for all configuration variables.
This table will also have information about where the current value came
from and some additional information about the variable. The historical
configuration files can be used as before.

This WL provides a read only system variable named persisted-globals-load
which provides an option to enable/disable reading of persistent config file.

client/base/abstract_option.h

@@ -138,7 +138,6 @@ template<typename T_type> Abstract_option<T_type>::Abstract_option(void* value,
   this->m_option_structure.block_size= 0;
   this->m_option_structure.max_value= 0;
   this->m_option_structure.min_value= 0;
-  this->m_option_structure.sub_size= 0;
   this->m_option_structure.typelib= NULL;
   this->m_option_structure.u_max_value= NULL;
 
@@ -162,6 +161,7 @@ template<typename T_type> Abstract_option<T_type>::Abstract_option(void* value,
 
   this->m_option_structure.value= value;
   this->m_option_structure.var_type= var_type;
+  this->m_option_structure.arg_source= 0;
 }
 
 template<typename T_type> my_option Abstract_option<T_type>::get_my_option()

client/mysql.cc

@@ -1836,11 +1836,11 @@ static struct my_option my_long_options[] =
    "The maximum packet length to send to or receive from server.",
    &opt_max_allowed_packet, &opt_max_allowed_packet, 0,
    GET_ULONG, REQUIRED_ARG, 16 *1024L*1024L, 4096,
-   (longlong) 2*1024L*1024L*1024L, MALLOC_OVERHEAD, 1024, 0},
+   (longlong) 2*1024L*1024L*1024L, 0, 1024, 0},
   {"net_buffer_length", OPT_NET_BUFFER_LENGTH,
    "The buffer size for TCP/IP and socket communication.",
    &opt_net_buffer_length, &opt_net_buffer_length, 0, GET_ULONG,
-   REQUIRED_ARG, 16384, 1024, 512*1024*1024L, MALLOC_OVERHEAD, 1024, 0},
+   REQUIRED_ARG, 16384, 1024, 512*1024*1024L, 0, 1024, 0},
   {"select_limit", OPT_SELECT_LIMIT,
    "Automatic limit for SELECT when using --safe-updates.",
    &select_limit, &select_limit, 0, GET_ULONG, REQUIRED_ARG, 1000L,

client/mysqlbinlog.cc

@@ -1682,7 +1682,7 @@ static struct my_option my_long_options[] =
    &opt_binlog_rows_event_max_size, 0,
    GET_ULONG, REQUIRED_ARG,
    /* def_value 4GB */ UINT_MAX, /* min_value */ 256,
-   /* max_value */ ULONG_MAX, /* sub_size */ 0,
+   /* max_value */ ULONG_MAX, /* arg_source */ 0,
    /* block_size */ 256, /* app_type */ 0},
   {"skip-gtids", OPT_MYSQLBINLOG_SKIP_GTIDS,
    "Do not preserve Global Transaction Identifiers; instead make the server "

client/mysqldump.cc

@@ -425,12 +425,12 @@ static struct my_option my_long_options[] =
    "The maximum packet length to send to or receive from server.",
     &opt_max_allowed_packet, &opt_max_allowed_packet, 0,
     GET_ULONG, REQUIRED_ARG, 24*1024*1024, 4096,
-   (longlong) 2L*1024L*1024L*1024L, MALLOC_OVERHEAD, 1024, 0},
+   (longlong) 2L*1024L*1024L*1024L, 0, 1024, 0},
   {"net_buffer_length", OPT_NET_BUFFER_LENGTH, 
    "The buffer size for TCP/IP and socket communication.",
     &opt_net_buffer_length, &opt_net_buffer_length, 0,
     GET_ULONG, REQUIRED_ARG, 1024*1024L-1025, 4096, 16*1024L*1024L,
-   MALLOC_OVERHEAD-1024, 1024, 0},
+   0, 1024, 0},
   {"no-autocommit", OPT_AUTOCOMMIT,
    "Wrap tables with autocommit/commit statements.",
    &opt_autocommit, &opt_autocommit, 0, GET_BOOL, NO_ARG,

configure.cmake

@@ -129,7 +129,10 @@ MACRO(EXTEND_C_LINK_FLAGS LIBRARY_PATH)
     "${CMAKE_SHARED_LIBRARY_C_FLAGS} -R'\$ORIGIN/../lib' -R${LIBRARY_PATH}")
 ENDMACRO()
 
-IF(CMAKE_SYSTEM_NAME MATCHES "SunOS" AND CMAKE_COMPILER_IS_GNUCC)
+# We assume that the client code is built with -std=c++11
+# which means that the necessary gcc libraries must be available.
+# Hence we disable the extra link flags, as well as installation of libraries.
+IF(CMAKE_SYSTEM_NAME MATCHES "SunOS" AND DISABLED_CMAKE_COMPILER_IS_GNUCC)
   DIRNAME(${CMAKE_CXX_COMPILER} CXX_PATH)
   SET(LIB_SUFFIX "lib")
   IF(SIZEOF_VOIDP EQUAL 8 AND CMAKE_SYSTEM_PROCESSOR MATCHES "sparc")

include/my_default.h

@@ -28,10 +28,11 @@ extern const char *my_defaults_extra_file;
 extern const char *my_defaults_group_suffix;
 extern const char *my_defaults_file;
 extern my_bool my_getopt_use_args_separator;
+extern my_bool no_defaults;
 
 /* Define the type of function to be passed to process_default_option_files */
 typedef int (*Process_option_func)(void *ctx, const char *group_name,
-                                   const char *option);
+                                   const char *option, const char *cnf_file);
 
 my_bool my_getopt_is_args_separator(const char* arg);
 int get_defaults_options(int argc, char **argv,
@@ -51,6 +52,9 @@ int my_search_option_files(const char *conf_file, int *argc,
 void free_defaults(char **argv);
 void my_print_default_files(const char *conf_file);
 void print_defaults(const char *conf_file, const char **groups);
+void init_variable_default_paths();
+void update_variable_source(const char* opt_name, const char* config_file);
+void set_variable_source(const char *opt_name, void* value);
 
 C_MODE_END
 

include/my_getopt.h

@@ -64,6 +64,36 @@ C_MODE_START
 */
 enum get_opt_arg_type { NO_ARG, OPT_ARG, REQUIRED_ARG };
 
+/**
+  Enumeration of the my_option::arg_source.m_source attribute. This enum values
+  define how system variables are set. For example if a variable is set by
+  global option file /etc/my.cnf then my_option::arg_source.m_source
+  will be set to GLOBAL, or if a variable is set from command line then
+  my_option::arg_source.m_source will hold value as COMMAND_LINE.
+*/
+enum enum_variable_source
+{
+  COMPILED= 1,
+  GLOBAL,
+  SERVER,
+  EXPLICIT,
+  EXTRA,
+  MYSQL_USER,
+  LOGIN,
+  COMMAND_LINE,
+  PERSISTED,
+  DYNAMIC
+};
+
+struct get_opt_arg_source
+{
+  /**
+    config file path OR compiled default values
+  */
+  const char* m_name;
+  enum enum_variable_source m_source;
+};
+
 struct st_typelib;
 
 struct my_option
@@ -99,7 +129,8 @@ struct my_option
   longlong   def_value;                 /**< Default value */
   longlong   min_value;                 /**< Min allowed value (for numbers) */
   ulonglong  max_value;                 /**< Max allowed value (for numbers) */
-  longlong   sub_size;                  /**< Unused                          */
+  struct get_opt_arg_source *arg_source;/**< Represents source/path from where this
+                                             variable is set. */
   long       block_size;                /**< Value should be a mult. of this (for numbers) */
   void       *app_type;                 /**< To be used by an application */
 };
@@ -144,6 +175,8 @@ ulonglong max_of_int_range(int var_type);
 
 ulonglong getopt_double2ulonglong(double);
 double getopt_ulonglong2double(ulonglong);
+int findopt(char *, uint, const struct my_option **);
+
 
 C_MODE_END
 

include/mysql_version.h.in

@@ -17,6 +17,7 @@
 #define MYSQL_PORT_DEFAULT          @MYSQL_TCP_PORT_DEFAULT@
 #define MYSQL_UNIX_ADDR            "@MYSQL_UNIX_ADDR@"
 #define MYSQL_CONFIG_NAME          "my"
+#define MYSQL_PERSIST_CONFIG_NAME  "mysqld-auto"
 #define MYSQL_COMPILATION_COMMENT  "@COMPILATION_COMMENT@"
 #define LIBMYSQL_VERSION           "@VERSION@"
 #define LIBMYSQL_VERSION_ID         @MYSQL_VERSION_ID@

libmysqld/lib_sql.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)  2000, 2014
+ * Copyright (c)  2000, 2016
  * SWsoft  company
  *
  * Modifications copyright (c) 2001, 2016. Oracle and/or its affiliates.
@@ -30,6 +30,7 @@
 #include "mysqld.h"
 #include "mysqld_embedded.h"
 #include "mysqld_thd_manager.h"
+#include "persisted_variable.h"         // Persisted_variables_cache
 #include "rpl_filter.h"
 #include "sql_class.h"
 #include "sql_db.h"
@@ -483,6 +484,9 @@ MYSQL_METHODS embedded_methods=
 
 char **		copy_arguments_ptr= 0;
 
+/* cache for persisted variables */
+static Persisted_variables_cache persisted_variables_cache;
+
 int init_embedded_server(int argc, char **argv, char **groups)
 {
   /*
@@ -537,11 +541,16 @@ int init_embedded_server(int argc, char **argv, char **groups)
   orig_argv= *argvp;
   if (load_defaults("my", (const char **)groups, argcp, argvp))
     return 1;
+
+  /* Initialize variables cache for persisted variables */
+  persisted_variables_cache.init();
+
   defaults_argc= *argcp;
   defaults_argv= *argvp;
   remaining_argc= *argcp;
   remaining_argv= *argvp;
 
+  init_variable_default_paths();
   /* Must be initialized early for comparison of options name */
   system_charset_info= &my_charset_utf8_general_ci;
   sys_var_init();
@@ -654,6 +663,13 @@ int init_embedded_server(int argc, char **argv, char **groups)
     check_performance_schema();
   }
 
+  /* set all persistent options */
+  if (persisted_variables_cache.set_persist_options())
+  {
+    sql_print_error("Setting persistent options failed.");
+    return 1;
+  }
+
   start_handle_manager();
 
   // FIXME initialize binlog_filter and rpl_filter if not already done

mysql-test/r/mysqld--help-notwin.result

@@ -828,6 +828,11 @@ The following options may be given as the first argument:
  --performance-schema-users-size=# 
  Maximum number of instrumented users. Use 0 to disable,
  -1 for automated scaling.
+ --persisted-globals-load 
+ When this option is enabled, config file mysqld-auto.cnf
+ is read and applied to server, else this file is ignored
+ even if present.
+ (Defaults to on; use --skip-persisted-globals-load to disable.)
  --pid-file=name     Pid file used by safe_mysqld
  --plugin-dir=name   Directory for plugins
  --plugin-load=name  Optional semicolon-separated list of plugins to load,
@@ -1439,7 +1444,7 @@ performance-schema-consumer-global-instrumentation TRUE
 performance-schema-consumer-statements-digest TRUE
 performance-schema-consumer-thread-instrumentation TRUE
 performance-schema-digests-size -1
-performance-schema-error-size 1037
+performance-schema-error-size 1038
 performance-schema-events-stages-history-long-size -1
 performance-schema-events-stages-history-size -1
 performance-schema-events-statements-history-long-size -1
@@ -1480,6 +1485,7 @@ performance-schema-session-connect-attrs-size -1
 performance-schema-setup-actors-size -1
 performance-schema-setup-objects-size -1
 performance-schema-users-size -1
+persisted-globals-load TRUE
 port ####
 port-open-timeout 0
 preload-buffer-size 32768

mysql-test/r/mysqld--help-win.result

@@ -819,6 +819,11 @@ The following options may be given as the first argument:
  --performance-schema-users-size=# 
  Maximum number of instrumented users. Use 0 to disable,
  -1 for automated scaling.
+ --persisted-globals-load 
+ When this option is enabled, config file mysqld-auto.cnf
+ is read and applied to server, else this file is ignored
+ even if present.
+ (Defaults to on; use --skip-persisted-globals-load to disable.)
  --pid-file=name     Pid file used by safe_mysqld
  --plugin-dir=name   Directory for plugins
  --plugin-load=name  Optional semicolon-separated list of plugins to load,
@@ -1435,7 +1440,7 @@ performance-schema-consumer-global-instrumentation TRUE
 performance-schema-consumer-statements-digest TRUE
 performance-schema-consumer-thread-instrumentation TRUE
 performance-schema-digests-size -1
-performance-schema-error-size 1037
+performance-schema-error-size 1038
 performance-schema-events-stages-history-long-size -1
 performance-schema-events-stages-history-size -1
 performance-schema-events-statements-history-long-size -1
@@ -1476,6 +1481,7 @@ performance-schema-session-connect-attrs-size -1
 performance-schema-setup-actors-size -1
 performance-schema-setup-objects-size -1
 performance-schema-users-size -1
+persisted-globals-load TRUE
 port ####
 port-open-timeout 0
 preload-buffer-size 32768