Bug#31586243: SERVER CRASH WITH BAD_ALLOC WHEN MEMORY IS TIGHT IN GR

Rpl_transaction_write_set_ctx::add_write_set may throw bad_alloc
during vector update.

Resolution: Code has been modified to handle bad_alloc from
add_write_set.

ReviewBoard: 24813

Author: Jaideep Karande

mysql-test/suite/group_replication/r/gr_bad_alloc_during_write_set.result

@@ -0,0 +1,25 @@
+include/group_replication.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection server1]
+CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+
+# 1. Set DBUG_SYNC to throw bad_alloc.
+SET @debug_saved= @@GLOBAL.DEBUG;
+SET @@GLOBAL.DEBUG= '+d,add_write_set_crash_no_memory';
+
+# 2. Confirm there is no crash.
+INSERT INTO t1 VALUES (2);
+ERROR HY000: Out of memory; check if mysqld or some other process uses all available memory; if not, you may have to use 'ulimit' to allow mysqld to use more memory or you can add more swap space
+SET @@GLOBAL.DEBUG= @debug_saved;
+
+# 3. Data is consistem on M1 and M2.
+INSERT INTO t1 VALUES (3);
+include/rpl_sync.inc
+include/diff_tables.inc [server_1:t1, server2:t1]
+
+# 4. Cleanup.
+DROP TABLE t1;
+include/group_replication_end.inc

mysql-test/suite/group_replication/t/gr_bad_alloc_during_write_set.test

@@ -0,0 +1,43 @@
+################################################################################
+# This test verify that when memory allocation fails during write set
+# push back, it is handled properly.
+#
+# Test:
+# 0. The test requires two servers: M1 and M2.
+# 1. Set DBUG_SYNC to throw bad_alloc.
+# 2. Confirm there is no crash.
+# 3. Data is consistem on M1 and M2.
+# 4. Cleanup.
+################################################################################
+--source include/have_debug.inc
+--source include/have_group_replication_plugin.inc
+--source include/group_replication.inc
+
+CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+
+--echo
+--echo # 1. Set DBUG_SYNC to throw bad_alloc.
+SET @debug_saved= @@GLOBAL.DEBUG;
+SET @@GLOBAL.DEBUG= '+d,add_write_set_crash_no_memory';
+
+--echo
+--echo # 2. Confirm there is no crash.
+
+--error 1041
+INSERT INTO t1 VALUES (2);
+SET @@GLOBAL.DEBUG= @debug_saved;
+
+--echo
+--echo # 3. Data is consistem on M1 and M2.
+
+INSERT INTO t1 VALUES (3);
+--source include/rpl_sync.inc
+
+--let $diff_tables=server_1:t1, server2:t1
+--source include/diff_tables.inc
+
+--echo
+--echo # 4. Cleanup.
+DROP TABLE t1;
+--source include/group_replication_end.inc

sql/handler.cc

@@ -7686,12 +7686,17 @@ int binlog_log_row(TABLE *table, const uchar *before_record,
 
   if (check_table_binlog_row_based(thd, table)) {
     if (thd->variables.transaction_write_set_extraction != HASH_ALGORITHM_OFF) {
-      if (before_record && after_record) {
-        /* capture both images pke */
-        add_pke(table, thd, table->record[0]);
-        add_pke(table, thd, table->record[1]);
-      } else {
-        add_pke(table, thd, table->record[0]);
+      try {
+        if (before_record && after_record) {
+          /* capture both images pke */
+          add_pke(table, thd, table->record[0]);
+          add_pke(table, thd, table->record[1]);
+        } else {
+          add_pke(table, thd, table->record[0]);
+        }
+      } catch (const std::bad_alloc &) {
+        my_error(ER_OUT_OF_RESOURCES, MYF(0));
+        return HA_ERR_RBR_LOGGING_FAILED;
       }
     }
     if (table->in_use->is_error()) return error ? HA_ERR_RBR_LOGGING_FAILED : 0;

sql/rpl_transaction_write_set_ctx.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -51,6 +51,7 @@ Rpl_transaction_write_set_ctx::Rpl_transaction_write_set_ctx()
 
 void Rpl_transaction_write_set_ctx::add_write_set(uint64 hash) {
   DBUG_TRACE;
+  DBUG_EXECUTE_IF("add_write_set_crash_no_memory", throw std::bad_alloc(););
   write_set.push_back(hash);
 }