Drafting mysql_hostgroup_attributes

include/MySQL_HostGroups_Manager.h

@@ -18,6 +18,14 @@
 
 #include "ev.h"
 
+#ifndef SPOOKYV2
+#include "SpookyV2.h"
+#define SPOOKYV2
+#endif
+
+#include "../deps/json/json.hpp"
+using json = nlohmann::json;
+
 #ifdef DEBUG
 /* */
 //	Enabling STRESSTEST_POOL ProxySQL will do a lot of loops in the connection pool
@@ -56,6 +64,9 @@
 
 #define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port"
 
+#define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
+
+
 typedef std::unordered_map<std::uint64_t, void *> umap_mysql_errors;
 
 class MySrvConnList;
@@ -197,6 +208,21 @@ class MyHGC {	// MySQL Host Group Container
 	unsigned long long current_time_now;
 	uint32_t new_connections_now;
 	MySrvList *mysrvs;
+	struct { // this is a series of attributes specific for each hostgroup
+		char * init_connect;
+		char * comment;
+		char * ignore_session_variables_text; // this is the original version (text format) of ignore_session_variables
+		uint32_t max_num_online_servers;
+		uint32_t throttle_connections_per_sec;
+		int8_t autocommit;
+		int8_t free_connections_pct;
+		bool multiplex;
+		bool connection_warming;
+		bool configured; // this variable controls if attributes are configured or not. If not configured, they do not apply
+		bool initialized; // this variable controls if attributes were ever configured or not. Used by reset_attributes()
+		json ignore_session_variables_json; // the JSON format of ignore_session_variables
+	} attributes;
+	void reset_attributes();
 	MyHGC(int);
 	~MyHGC();
 	MySrvC *get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, MySQL_Session *sess);
@@ -404,6 +430,7 @@ class MySQL_HostGroups_Manager {
 	 *   - 'CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS'
 	 *   - 'CLUSTER_QUERY_MYSQL_GALERA'
 	 *   - 'CLUSTER_QUERY_MYSQL_AWS_AURORA'
+	 *   - 'CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES'
 	 *  Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets.
 	 */
 	SQLite3_result *incoming_replication_hostgroups;
@@ -426,6 +453,9 @@ class MySQL_HostGroups_Manager {
 	pthread_mutex_t AWS_Aurora_Info_mutex;
 	std::map<int , AWS_Aurora_Info *> AWS_Aurora_Info_Map;
 
+	void generate_mysql_hostgroup_attributes_table();
+	SQLite3_result *incoming_hostgroup_attributes;
+
 	std::thread *HGCU_thread;
 
 	std::thread *GTID_syncer_thread;
@@ -570,6 +600,8 @@ class MySQL_HostGroups_Manager {
 	void wrunlock();
 	int servers_add(SQLite3_result *resultset);
 	bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0);
+	void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init);
+	void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName); // used by commit_update_checksums_from_tables()
 
 	/**
 	 * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime.
@@ -583,23 +615,13 @@ class MySQL_HostGroups_Manager {
 	 *  Cluster to propagate current config.
 	 * @param The resulset to be stored replacing the current one.
 	 */
-	void save_incoming_replication_hostgroups(SQLite3_result *);
-	void save_incoming_group_replication_hostgroups(SQLite3_result *);
-	void save_incoming_galera_hostgroups(SQLite3_result *);
-	void save_incoming_aws_aurora_hostgroups(SQLite3_result *);
 
-	SQLite3_result* get_current_mysql_servers_inner();
-	SQLite3_result* get_current_mysql_replication_hostgroups_inner();
-	SQLite3_result* get_current_mysql_group_replication_hostgroups_inner();
-	SQLite3_result* get_current_mysql_galera_hostgroups();
-	SQLite3_result* get_current_mysql_aws_aurora_hostgroups();
+	void save_incoming_mysql_table(SQLite3_result *, const string&);
+	SQLite3_result* get_current_mysql_table(const string& name);
 
 	SQLite3_result * execute_query(char *query, char **error);
-	SQLite3_result *dump_table_mysql_servers();
-	SQLite3_result *dump_table_mysql_replication_hostgroups();
-	SQLite3_result *dump_table_mysql_group_replication_hostgroups();
-	SQLite3_result *dump_table_mysql_galera_hostgroups();
-	SQLite3_result *dump_table_mysql_aws_aurora_hostgroups();
+	SQLite3_result *dump_table_mysql(const string&);
+
 	/**
 	 * @brief Update the public member resulset 'mysql_servers_to_monitor'. This resulset should contain the latest
 	 *   'mysql_servers' present in 'MySQL_HostGroups_Manager' db, which are not 'OFFLINE_HARD'. The resulset

include/ProxySQL_Cluster.hpp

@@ -20,7 +20,7 @@
  * also represent the actual resultset being received when issuing them, since this resultset is used for computing the
  * 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules:
  *   - 'runtime_mysql_servers': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups',
- *     'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups'.
+ *     'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'.
  *   - 'runtime_mysql_users'.
  *   - 'runtime_mysql_query_rules'.
  *
@@ -37,6 +37,9 @@
 /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_group_replication_hostgroups'. See top comment for details. */
 #define CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup"
 
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_hostgroup_attributes'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES "PROXY_SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment FROM runtime_mysql_hostgroup_attributes ORDER BY hostgroup_id"
+
 /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */
 #define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"
 
@@ -299,6 +302,8 @@ struct p_cluster_counter {
 		pulled_mysql_servers_galera_hostgroups_failure,
 		pulled_mysql_servers_aws_aurora_hostgroups_success,
 		pulled_mysql_servers_aws_aurora_hostgroups_failure,
+		pulled_mysql_servers_hostgroup_attributes_success,
+		pulled_mysql_servers_hostgroup_attributes_failure,
 		pulled_mysql_servers_runtime_checks_success,
 		pulled_mysql_servers_runtime_checks_failure,
 

include/mysql_connection.h

@@ -17,7 +17,7 @@ using json = nlohmann::json;
 #define STATUS_MYSQL_CONNECTION_NO_MULTIPLEX         0x00000080
 #define STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0         0x00000100
 #define STATUS_MYSQL_CONNECTION_FOUND_ROWS           0x00000200
-//#define STATUS_MYSQL_CONNECTION_NO_BACKSLASH_ESCAPES 0x00000400
+#define STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG      0x00000400
 #define STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT        0x00000800
 
 class Variable {

include/proxysql_admin.h

@@ -122,9 +122,10 @@ struct incoming_servers_t {
 	SQLite3_result* incoming_group_replication_hostgroups = NULL;
 	SQLite3_result* incoming_galera_hostgroups = NULL;
 	SQLite3_result* incoming_aurora_hostgroups = NULL;
+	SQLite3_result* incoming_hostgroup_attributes = NULL;
 
 	incoming_servers_t();
-	incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*);
+	incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*);
 };
 
 class ProxySQL_Admin {
@@ -357,14 +358,11 @@ class ProxySQL_Admin {
 	int load_debug_to_runtime();
 	void save_debug_from_runtime();
 #endif // DEBUG
+
+	void flush_GENERIC__from_to(const std::string&, const std::string&);
+
 	void flush_mysql_users__from_memory_to_disk();
 	void flush_mysql_users__from_disk_to_memory();
-	void flush_mysql_servers__from_memory_to_disk();
-	void flush_mysql_servers__from_disk_to_memory();
-	void flush_mysql_query_rules__from_memory_to_disk();
-	void flush_mysql_query_rules__from_disk_to_memory();
-	void flush_mysql_firewall__from_memory_to_disk();
-	void flush_mysql_firewall__from_disk_to_memory();
 
 //	void flush_mysql_variables__from_disk_to_memory(); // commented in 2.3 because unused
 	void flush_mysql_variables__from_memory_to_disk();
@@ -408,8 +406,6 @@ class ProxySQL_Admin {
 
 	void load_scheduler_to_runtime();
 	void save_scheduler_runtime_to_database(bool);
-	void flush_scheduler__from_memory_to_disk();
-	void flush_scheduler__from_disk_to_memory();
 
 	void load_admin_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0) { flush_admin_variables___database_to_runtime(admindb, true, checksum, epoch); }
 	void save_admin_variables_from_runtime() { flush_admin_variables___runtime_to_database(admindb, true, true, false); }
@@ -463,8 +459,6 @@ class ProxySQL_Admin {
 
 	// Cluster
 	void load_proxysql_servers_to_runtime(bool _lock=true, const std::string& checksum = "", const time_t epoch = 0);
-	void flush_proxysql_servers__from_memory_to_disk();
-	void flush_proxysql_servers__from_disk_to_memory();
 	void save_proxysql_servers_runtime_to_database(bool);
 	void dump_checksums_values_table();
 

lib/ClickHouse_Authentication.cpp

@@ -3,7 +3,10 @@
 #include "proxysql.h"
 #include "cpp.h"
 #include "proxysql_atomic.h"
+#ifndef SPOOKYV2
 #include "SpookyV2.h"
+#define SPOOKYV2
+#endif
 
 ClickHouse_Authentication::ClickHouse_Authentication() {
 #ifdef DEBUG

lib/ClickHouse_Server.cpp

@@ -23,7 +23,10 @@
 #include <resolv.h>
 #include <arpa/inet.h>
 #include <pthread.h>
+#ifndef SPOOKYV2
 #include "SpookyV2.h"
+#define SPOOKYV2
+#endif
 
 #include <fcntl.h>
 #include <sys/utsname.h>

lib/MySQL_Authentication.cpp

@@ -5,7 +5,10 @@
 
 #include "MySQL_Authentication.hpp"
 
+#ifndef SPOOKYV2
 #include "SpookyV2.h"
+#define SPOOKYV2
+#endif
 
 MySQL_Authentication::MySQL_Authentication() {
 #ifdef DEBUG