MDEV-9519: Data corruption will happen on the Galera cluster size change

If we have a 2+ node cluster which is replicating from an async master
and the binlog_format is set to STATEMENT and multi-row inserts are executed
on a table with an auto_increment column such that values are automatically
generated by MySQL, then the server node generates wrong auto_increment
values, which are different from what was generated on the async master.

In the title of the MDEV-9519 it was proposed to ban start slave on a Galera
if master binlog_format = statement and wsrep_auto_increment_control = 1,
but the problem can be solved without such a restriction.

The causes and fixes:

1. We need to improve processing of changing the auto-increment values
after changing the cluster size.

2. If wsrep auto_increment_control switched on during operation of
the node, then we should immediately update the auto_increment_increment
and auto_increment_offset global variables, without waiting of the next
invocation of the wsrep_view_handler_cb() callback. In the current version
these variables retain its initial values if wsrep_auto_increment_control
is switched on during operation of the node, which leads to inconsistent
results on the different nodes in some scenarios.

3. If wsrep auto_increment_control switched off during operation of the node,
then we must return the original values of the auto_increment_increment and
auto_increment_offset global variables, as the user has set. To make this
possible, we need to add a "shadow copies" of these variables (which stores
the latest values set by the user).

https://jira.mariadb.org/browse/MDEV-9519

include/mysql/service_wsrep.h

@@ -107,6 +107,7 @@ extern struct wsrep_service_st {
   bool                        (*wsrep_thd_ignore_table_func)(THD *thd);
   long long                   (*wsrep_thd_trx_seqno_func)(THD *thd);
   struct wsrep_ws_handle *    (*wsrep_thd_ws_handle_func)(THD *thd);
+  void                        (*wsrep_thd_auto_increment_variables_func)(THD *thd, unsigned long long *offset, unsigned long long *increment);
   int                         (*wsrep_trx_is_aborting_func)(MYSQL_THD thd);
   int                         (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD);
   void                        (*wsrep_unlock_rollback_func)();
@@ -149,6 +150,7 @@ extern struct wsrep_service_st {
 #define wsrep_thd_ignore_table(T) wsrep_service->wsrep_thd_ignore_table_func(T)
 #define wsrep_thd_trx_seqno(T) wsrep_service->wsrep_thd_trx_seqno_func(T)
 #define wsrep_thd_ws_handle(T) wsrep_service->wsrep_thd_ws_handle_func(T)
+#define wsrep_thd_auto_increment_variables(T,O,I) wsrep_service->wsrep_thd_auto_increment_variables_func(T,O,I)
 #define wsrep_trx_is_aborting(T) wsrep_service->wsrep_trx_is_aborting_func(T)
 #define wsrep_trx_order_before(T1,T2) wsrep_service->wsrep_trx_order_before_func(T1,T2)
 #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func()
@@ -201,6 +203,7 @@ my_bool wsrep_thd_is_BF(MYSQL_THD thd, my_bool sync);
 my_bool wsrep_thd_is_wsrep(MYSQL_THD thd);
 struct wsrep *get_wsrep();
 struct wsrep_ws_handle *wsrep_thd_ws_handle(THD *thd);
+void wsrep_thd_auto_increment_variables(THD *thd, unsigned long long *offset, unsigned long long *increment);
 void wsrep_aborting_thd_enqueue(THD *thd);
 void wsrep_lock_rollback();
 void wsrep_post_commit(THD* thd, bool all);

mysql-test/suite/galera/disabled.def

@@ -25,7 +25,6 @@ galera.MW-329 : wsrep_local_replays not stable
 MW-416 : MDEV-13549 Galera test failures
 MW-388 : MDEV-13549 Galera test failures
 galera_sst_mysqldump_with_key : MDEV-16890 Galera test failure
-galera_binlog_stmt_autoinc: MDEV-17106 Test failure on galera.galera_binlog_stmt_autoinc
 galera_gc_fc_limit : MDEV-17061 Test failure on galera.galera_gc_fc_limit
 galera_as_slave_replication_budle : MDEV-15785 Test case galera_as_slave_replication_bundle caused debug assertion
 galera_wan : MDEV-17259: Test failure on galera.galera_wan

mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result

@@ -8,20 +8,20 @@ PRIMARY KEY (i)
 insert into t1(i) values(null);
 select * from t1;
 i	c
-3	dummy_text
+1	dummy_text
 insert into t1(i) values(null), (null), (null);
 select * from t1;
 i	c
+1	dummy_text
 3	dummy_text
 5	dummy_text
 7	dummy_text
-9	dummy_text
 select * from t1;
 i	c
+1	dummy_text
 3	dummy_text
 5	dummy_text
 7	dummy_text
-9	dummy_text
 SET GLOBAL wsrep_forced_binlog_format='none';
 SET GLOBAL wsrep_forced_binlog_format='none';
 drop table t1;
@@ -40,20 +40,20 @@ PRIMARY KEY (i)
 insert into t1(i) values(null);
 select * from t1;
 i	c
-4	dummy_text
+1	dummy_text
 insert into t1(i) values(null), (null), (null);
 select * from t1;
 i	c
+1	dummy_text
 4	dummy_text
 7	dummy_text
 10	dummy_text
-13	dummy_text
 select * from t1;
 i	c
+1	dummy_text
 4	dummy_text
 7	dummy_text
 10	dummy_text
-13	dummy_text
 SET GLOBAL wsrep_auto_increment_control='ON';
 SET SESSION binlog_format='ROW';
 show variables like 'binlog_format';
@@ -67,7 +67,7 @@ wsrep_auto_increment_control	ON
 SET GLOBAL wsrep_auto_increment_control='OFF';
 show variables like '%auto_increment%';
 Variable_name	Value
-auto_increment_increment	2
+auto_increment_increment	3
 auto_increment_offset	1
 wsrep_auto_increment_control	OFF
 SET GLOBAL wsrep_auto_increment_control='ON';
@@ -82,20 +82,20 @@ PRIMARY KEY (i)
 insert into t1(i) values(null);
 select * from t1;
 i	c
-3	dummy_text
+1	dummy_text
 insert into t1(i) values(null), (null), (null);
 select * from t1;
 i	c
+1	dummy_text
 3	dummy_text
 5	dummy_text
 7	dummy_text
-9	dummy_text
 select * from t1;
 i	c
+1	dummy_text
 3	dummy_text
 5	dummy_text
 7	dummy_text
-9	dummy_text
 SET GLOBAL wsrep_forced_binlog_format='none';
 SET GLOBAL wsrep_forced_binlog_format='none';
 drop table t1;
@@ -114,20 +114,20 @@ PRIMARY KEY (i)
 insert into t1(i) values(null);
 select * from t1;
 i	c
-4	dummy_text
+1	dummy_text
 insert into t1(i) values(null), (null), (null);
 select * from t1;
 i	c
+1	dummy_text
 4	dummy_text
 7	dummy_text
 10	dummy_text
-13	dummy_text
 select * from t1;
 i	c
+1	dummy_text
 4	dummy_text
 7	dummy_text
 10	dummy_text
-13	dummy_text
 SET GLOBAL wsrep_auto_increment_control='ON';
 show variables like 'binlog_format';
 Variable_name	Value
@@ -140,7 +140,7 @@ wsrep_auto_increment_control	ON
 SET GLOBAL wsrep_auto_increment_control='OFF';
 show variables like '%auto_increment%';
 Variable_name	Value
-auto_increment_increment	2
+auto_increment_increment	3
 auto_increment_offset	1
 wsrep_auto_increment_control	OFF
 SET GLOBAL wsrep_auto_increment_control='ON';

mysql-test/suite/parts/r/partition_auto_increment_max.result

@@ -0,0 +1,7 @@
+CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2;
+INSERT INTO t1 VALUES (NULL),(NULL);
+UPDATE t1 SET pk = 2147483647;
+ERROR 23000: Duplicate entry '2147483647' for key 'PRIMARY'
+REPLACE INTO t1 VALUES (NULL);
+ERROR 22003: Out of range value for column 'pk' at row 1
+DROP TABLE t1;

mysql-test/suite/parts/t/partition_auto_increment_max.test

@@ -0,0 +1,12 @@
+--source include/have_partition.inc
+--source include/have_log_bin.inc
+
+CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) PARTITION BY KEY (pk) PARTITIONS 2;
+INSERT INTO t1 VALUES (NULL),(NULL);
+
+--error ER_DUP_ENTRY
+UPDATE t1 SET pk = 2147483647;
+--error HA_ERR_AUTOINC_ERANGE
+REPLACE INTO t1 VALUES (NULL);
+
+DROP TABLE t1;

sql/field.h

@@ -1329,6 +1329,17 @@ class Field: public Value_source
   /* Hash value */
   virtual void hash(ulong *nr, ulong *nr2);
 
+  /**
+    Get the upper limit of the MySQL integral and floating-point type.
+
+    @return maximum allowed value for the field
+  */
+  virtual ulonglong get_max_int_value() const
+  {
+    DBUG_ASSERT(false);
+    return 0ULL;
+  }
+
 /**
   Checks whether a string field is part of write_set.
 
@@ -1796,6 +1807,11 @@ class Field_tiny :public Field_num {
     *to= *from;
     return from + 1;
   }
+
+  virtual ulonglong get_max_int_value() const
+  {
+    return unsigned_flag ? 0xFFULL : 0x7FULL;
+  }
 };
 
 
@@ -1837,6 +1853,11 @@ class Field_short :public Field_num {
   virtual const uchar *unpack(uchar* to, const uchar *from,
                               const uchar *from_end, uint param_data)
   { return unpack_int16(to, from, from_end); }
+
+  virtual ulonglong get_max_int_value() const
+  {
+    return unsigned_flag ? 0xFFFFULL : 0x7FFFULL;
+  }
 };
 
 class Field_medium :public Field_num {
@@ -1870,6 +1891,11 @@ class Field_medium :public Field_num {
   {
     return Field::pack(to, from, max_length);
   }
+
+  virtual ulonglong get_max_int_value() const
+  {
+    return unsigned_flag ? 0xFFFFFFULL : 0x7FFFFFULL;
+  }
 };
 
 
@@ -1915,6 +1941,11 @@ class Field_long :public Field_num {
   {
     return unpack_int32(to, from, from_end);
   }
+
+  virtual ulonglong get_max_int_value() const
+  {
+    return unsigned_flag ? 0xFFFFFFFFULL : 0x7FFFFFFFULL;
+  }
 };
 
 
@@ -1964,6 +1995,10 @@ class Field_longlong :public Field_num {
   {
     return unpack_int64(to, from, from_end);
   }
+  virtual ulonglong get_max_int_value() const
+  {
+    return unsigned_flag ? 0xFFFFFFFFFFFFFFFFULL : 0x7FFFFFFFFFFFFFFFULL;
+  }
 };
 
 
@@ -1997,6 +2032,13 @@ class Field_float :public Field_real {
   uint32 pack_length() const { return sizeof(float); }
   uint row_pack_length() const { return pack_length(); }
   void sql_type(String &str) const;
+  virtual ulonglong get_max_int_value() const
+  {
+    /*
+      We use the maximum as per IEEE754-2008 standard, 2^24
+    */
+    return 0x1000000ULL;
+  }
 private:
   int do_save_field_metadata(uchar *first_byte);
 };
@@ -2039,6 +2081,13 @@ class Field_double :public Field_real {
   uint32 pack_length() const { return sizeof(double); }
   uint row_pack_length() const { return pack_length(); }
   void sql_type(String &str) const;
+  virtual ulonglong get_max_int_value() const
+  {
+    /*
+      We use the maximum as per IEEE754-2008 standard, 2^53
+    */
+    return 0x20000000000000ULL;
+  }
 private:
   int do_save_field_metadata(uchar *first_byte);
 };

sql/ha_partition.cc

@@ -8706,31 +8706,37 @@ void ha_partition::release_auto_increment()
       m_file[i]->ha_release_auto_increment();
     }
   }
-  else if (next_insert_id)
+  else
   {
-    ulonglong next_auto_inc_val;
     lock_auto_increment();
-    next_auto_inc_val= part_share->next_auto_inc_val;
-    /*
-      If the current auto_increment values is lower than the reserved
-      value, and the reserved value was reserved by this thread,
-      we can lower the reserved value.
-    */
-    if (next_insert_id < next_auto_inc_val &&
-        auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val)
+    if (next_insert_id)
     {
-      THD *thd= ha_thd();
+      ulonglong next_auto_inc_val= part_share->next_auto_inc_val;
       /*
-        Check that we do not lower the value because of a failed insert
-        with SET INSERT_ID, i.e. forced/non generated values.
+        If the current auto_increment values is lower than the reserved
+        value, and the reserved value was reserved by this thread,
+        we can lower the reserved value.
       */
-      if (thd->auto_inc_intervals_forced.maximum() < next_insert_id)
-        part_share->next_auto_inc_val= next_insert_id;
+      if (next_insert_id < next_auto_inc_val &&
+          auto_inc_interval_for_cur_row.maximum() >= next_auto_inc_val)
+      {
+        THD *thd= ha_thd();
+        /*
+          Check that we do not lower the value because of a failed insert
+          with SET INSERT_ID, i.e. forced/non generated values.
+        */
+        if (thd->auto_inc_intervals_forced.maximum() < next_insert_id)
+          part_share->next_auto_inc_val= next_insert_id;
+      }
+      DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu",
+                          (ulong) part_share->next_auto_inc_val));
     }
-    DBUG_PRINT("info", ("part_share->next_auto_inc_val: %lu",
-                        (ulong) part_share->next_auto_inc_val));
-
-    /* Unlock the multi row statement lock taken in get_auto_increment */
+    /*
+      Unlock the multi-row statement lock taken in get_auto_increment.
+      These actions must be performed even if the next_insert_id field
+      contains zero, otherwise if the update_auto_increment fails then
+      an unnecessary lock will remain:
+    */
     if (auto_increment_safe_stmt_log_lock)
     {
       auto_increment_safe_stmt_log_lock= FALSE;