MDEV-27246 Implement a method to add IPs to allowlist for Galera Clus…

…ter node addresses that can make SST/IST requests

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/galera/r/galera_defaults.result

@@ -20,6 +20,7 @@ AND VARIABLE_NAME NOT IN (
 )
 ORDER BY VARIABLE_NAME;
 VARIABLE_NAME	VARIABLE_VALUE
+WSREP_ALLOWLIST	
 WSREP_AUTO_INCREMENT_CONTROL	ON
 WSREP_CAUSAL_READS	ON
 WSREP_CERTIFICATION_RULES	strict

mysql-test/suite/galera/t/galera_defaults.test

@@ -13,13 +13,11 @@
 --source include/force_restart.inc
 
 # Make sure that the test is operating on the right version of galera library.
---let $galera_version=26.4.7
+--let $galera_version=26.4.11
 source ../wsrep/include/check_galera_version.inc;
 
 # Global Variables
 
-SELECT COUNT(*) `expect 50` FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_%';
-
 SELECT VARIABLE_NAME, VARIABLE_VALUE
 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
 WHERE VARIABLE_NAME LIKE 'wsrep_%'

mysql-test/suite/galera_3nodes/r/galera_allowlist.result

@@ -0,0 +1,35 @@
+connection node_2;
+connection node_1;
+SELECT COUNT(*) = 3 FROM mysql.wsrep_allowlist;
+COUNT(*) = 3
+1
+connection node_2;
+SELECT COUNT(*) = 3 FROM mysql.wsrep_allowlist;
+COUNT(*) = 3
+1
+connection node_3;
+SET @@global.wsrep_desync = 1;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
+connection node_1;
+DELETE FROM mysql.wsrep_allowlist WHERE ip LIKE '127.0.0.3';
+SELECT COUNT(*) = 2 FROM mysql.wsrep_allowlist;
+COUNT(*) = 2
+1
+connection node_2;
+SELECT COUNT(*) = 2 FROM mysql.wsrep_allowlist;
+COUNT(*) = 2
+1
+connection node_3;
+SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
+SET @@global.wsrep_desync = 0;
+connection node_1;
+INSERT INTO mysql.wsrep_allowlist(ip) VALUES ('127.0.0.3');
+connection node_3;
+# restart
+connection node_1;
+CALL mtr.add_suppression('WSREP: Connection not allowed');
+connection node_2;
+CALL mtr.add_suppression('WSREP: Connection not allowed');
+connection node_3;
+CALL mtr.add_suppression('WSREP: Ignoring lack of quorum');

mysql-test/suite/galera_3nodes/t/galera_allowlist.cnf

@@ -0,0 +1,26 @@
+!include ../galera_3nodes.cnf
+
+[mysqld]
+wsrep_sst_method=rsync
+
+[mysqld.1]
+wsrep_allowlist="127.0.0.1,127.0.0.2,127.0.0.3"
+
+[mysqld.2]
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;gmcast.listen_addr=127.0.0.2;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S'
+
+# Variable is only used on bootstrap node, so this will be ignored
+wsrep_allowlist="127.0.0.1,127.0.0.2,127.0.0.3,127.0.0.4,127.0.0.5"
+
+wsrep_node_address=127.0.0.2
+wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port
+wsrep_node_incoming_address=127.0.0.2:@mysqld.2.port
+wsrep_sst_receive_address='127.0.0.2:@mysqld.2.#sst_port'
+
+[mysqld.3]
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;gmcast.listen_addr=127.0.0.3;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_quorum=TRUE;pc.wait_prim=FALSE'
+
+wsrep_node_address=127.0.0.3
+wsrep_sst_receive_address=127.0.0.3:@mysqld.3.#sst_port
+wsrep_node_incoming_address=127.0.0.3:@mysqld.3.port
+wsrep_sst_receive_address='127.0.0.3:@mysqld.3.#sst_port'

mysql-test/suite/galera_3nodes/t/galera_allowlist.test

@@ -0,0 +1,66 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+# Check that `wsrep_allowlist` variable is loaded
+SELECT COUNT(*) = 3 FROM mysql.wsrep_allowlist;
+
+--connection node_2
+# Check that non-bootstrap nodes doesn't populate `mysql.wsrep_allowlist`
+SELECT COUNT(*) = 3 FROM mysql.wsrep_allowlist;
+
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+--connection node_3
+# Desync and disconnect node 3 from the PC:
+SET @@global.wsrep_desync = 1;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
+
+--connection node_1
+# Wait until node 3 disappears from the PC:
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# Delete node ip (127.0.0.3) from allowlist
+DELETE FROM mysql.wsrep_allowlist WHERE ip LIKE '127.0.0.3';
+
+SELECT COUNT(*) = 2 FROM mysql.wsrep_allowlist;
+
+--connection node_2
+SELECT COUNT(*) = 2 FROM mysql.wsrep_allowlist;
+
+--connection node_3
+# Reconnect node 2 to the PC:
+SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
+
+# We should reach Primary with cluster size = 1 because of `pc.ignore_quorum=TRUE and pc.wait_prim=FALSE` used in configuration
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# Resync should pass:
+SET @@global.wsrep_desync = 0;
+
+# Shutdown node
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+# Allow node 3 could be reconnected to cluster
+INSERT INTO mysql.wsrep_allowlist(ip) VALUES ('127.0.0.3');
+
+--connection node_3
+--source include/start_mysqld.inc
+--source include/wait_until_connected_again.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+CALL mtr.add_suppression('WSREP: Connection not allowed');
+
+--connection node_2
+CALL mtr.add_suppression('WSREP: Connection not allowed');
+
+--connection node_3
+CALL mtr.add_suppression('WSREP: Ignoring lack of quorum');

mysql-test/suite/sys_vars/r/sysvars_wsrep.result

@@ -1,6 +1,21 @@
 select * from information_schema.system_variables
 where variable_name like 'wsrep%'
   order by variable_name;
+VARIABLE_NAME	WSREP_ALLOWLIST
+SESSION_VALUE	NULL
+GLOBAL_VALUE	
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	VARCHAR
+VARIABLE_COMMENT	Allowed IP addresses split by comma delimiter
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	NULL
+READ_ONLY	YES
+COMMAND_LINE_ARGUMENT	REQUIRED
+GLOBAL_VALUE_PATH	NULL
 VARIABLE_NAME	WSREP_AUTO_INCREMENT_CONTROL
 SESSION_VALUE	NULL
 GLOBAL_VALUE	ON

mysql-test/suite/wsrep/r/variables.result

@@ -88,6 +88,7 @@ wsrep_thread_count	2
 # variables
 SELECT VARIABLE_NAME FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep%" ORDER BY VARIABLE_NAME;
 VARIABLE_NAME
+WSREP_ALLOWLIST
 WSREP_AUTO_INCREMENT_CONTROL
 WSREP_CAUSAL_READS
 WSREP_CERTIFICATION_RULES

mysql-test/suite/wsrep/r/variables_debug.result

@@ -89,6 +89,7 @@ wsrep_thread_count	2
 # variables
 SELECT VARIABLE_NAME FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep%" ORDER BY VARIABLE_NAME;
 VARIABLE_NAME
+WSREP_ALLOWLIST
 WSREP_AUTO_INCREMENT_CONTROL
 WSREP_CAUSAL_READS
 WSREP_CERTIFICATION_RULES

mysql-test/suite/wsrep/t/variables.test

@@ -3,7 +3,7 @@
 --source include/have_innodb.inc
 --source include/galera_no_debug_sync.inc
 
---let $galera_version=26.4.9
+--let $galera_version=26.4.11
 source include/check_galera_version.inc;
 
 source include/galera_variables_ok.inc;

mysql-test/suite/wsrep/t/variables_debug.test

@@ -5,7 +5,7 @@
 --source include/have_debug_sync.inc
 --source include/galera_have_debug_sync.inc
 
---let $galera_version=26.4.9
+--let $galera_version=26.4.11
 source include/check_galera_version.inc;
 
 source include/galera_variables_ok.inc;

sql/CMakeLists.txt

@@ -23,6 +23,7 @@ IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY)
     wsrep_storage_service.cc
     wsrep_server_state.cc
     wsrep_status.cc
+    wsrep_allowlist_service.cc
     wsrep_utils.cc
     wsrep_xid.cc
     wsrep_check_opts.cc

sql/sys_vars.cc

@@ -6236,6 +6236,12 @@ static Sys_var_charptr Sys_wsrep_patch_version(
        READ_ONLY GLOBAL_VAR(wsrep_patch_version_ptr), CMD_LINE_HELP_ONLY,
        DEFAULT(WSREP_PATCH_VERSION));
 
+
+static Sys_var_charptr Sys_wsrep_allowlist(
+       "wsrep_allowlist", "Allowed IP addresses split by comma delimiter",
+       READ_ONLY GLOBAL_VAR(wsrep_allowlist), CMD_LINE(REQUIRED_ARG),
+       DEFAULT(""));
+
 #endif /* WITH_WSREP */
 
 static bool fix_host_cache_size(sys_var *, THD *, enum_var_type)

sql/wsrep_allowlist_service.cc

@@ -0,0 +1,54 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include "wsrep_allowlist_service.h"
+
+#include "my_global.h"
+#include "wsrep_mysqld.h"
+#include "wsrep_priv.h"
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+class Wsrep_allowlist_service : public wsrep::allowlist_service
+{
+public:
+    bool allowlist_cb(wsrep::allowlist_service::allowlist_key key,
+                      const wsrep::const_buffer& value) WSREP_NOEXCEPT override;
+};
+
+bool Wsrep_allowlist_service::allowlist_cb (
+  wsrep::allowlist_service::allowlist_key key,
+  const wsrep::const_buffer& value)
+  WSREP_NOEXCEPT
+{
+  std::string string_value(value.data());
+  return (wsrep_schema->allowlist_check(key, string_value));
+}
+
+std::unique_ptr<wsrep::allowlist_service> entrypoint;
+
+wsrep::allowlist_service* wsrep_allowlist_service_init()
+{
+  entrypoint = std::unique_ptr<wsrep::allowlist_service>(new Wsrep_allowlist_service);
+  return entrypoint.get();
+}
+
+void wsrep_allowlist_service_deinit()
+{
+  entrypoint.reset();
+}
+

sql/wsrep_allowlist_service.h

@@ -0,0 +1,29 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+/*
+  Implementation of wsrep provider threads instrumentation.
+ */
+
+#ifndef WSREP_PROVIDER_ALLOWLIST_H
+#define WSREP_PROVIDER_ALLOWLIST_H
+
+#include "wsrep/allowlist_service.hpp"
+
+wsrep::allowlist_service* wsrep_allowlist_service_init();
+
+void wsrep_allowlist_service_deinit();
+
+#endif /* WSREP_PROVIDER_ALLOWLIST_H */