Skip to content

Commit d8962d1

Browse files
ThirunarayananThirunarayanan
committed
MDEV-36017 Alter table aborts when temporary directory is full
Problem: ======= - In 10.11, During Copy algorithm, InnoDB does use bulk insert for row by row insert operation. When temporary directory ran out of memory, row_mysql_handle_errors() fails to handle DB_TEMP_FILE_WRITE_FAIL. - During inplace algorithm, concurrent DML fails to write the log operation into the temporary file. InnoDB fail to mark the error for the online log. - ddl_log_write() releases the global ddl lock prematurely before release the log memory entry Fix: === row_mysql_handle_errors(): Rollback the transaction when InnoDB encounters DB_TEMP_FILE_WRITE_FAIL convert_error_code_to_mysql(): Report an aborted transaction when InnoDB encounters DB_TEMP_FILE_WRITE_FAIL during alter table algorithm=copy or innodb bulk insert operation row_log_online_op(): Mark the error in online log when InnoDB ran out of temporary space fil_space_extend_must_retry(): Mark the os_has_said_disk_full as true if os_file_set_size() fails btr_cur_pessimistic_update(): Return error code when btr_cur_pessimistic_insert() fails ddl_log_write(): Release the global ddl lock after releasing the log memory entry when error was encountered btr_cur_optimistic_update(): Relax the assertion that blob pointer can be null during rollback because InnoDB can ran out of space while allocating the external page ha_innobase::extra(): Rollback the transaction during DDL before calling convert_error_code_to_mysql(). row_undo_mod_upd_exist_sec(): Remove the assertion which says that InnoDB should fail to build index entry when rollbacking an incomplete transaction after crash recovery. This scenario can happen when InnoDB ran out of space. row_upd_changes_ord_field_binary_func(): Relax the assertion to make that externally stored field can be null when InnoDB ran out of space.
1 parent 8bc1643 commit d8962d1

File tree

12 files changed

+142
-31
lines changed

12 files changed

+142
-31
lines changed

‎mysql-test/suite/innodb/r/alloc_fail.result‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#
2+
# MDEV-36017 Alter table aborts when temporary
3+
# directory is full
4+
#
5+
SET SESSION DEFAULT_STORAGE_ENGINE=InnoDB;
6+
CREATE TABLE t1(f1 CHAR(100) NOT NULL, f2 CHAR(100) NOT NULL,
7+
f3 CHAR(100) NOT NULL, f4 CHAR(100) NOT NULL,
8+
f5 CHAR(100) NOT NULL)ENGINE=InnoDB;
9+
INSERT INTO t1 SELECT 'a', 'b', 'c', 'd', 'e' FROM seq_1_to_65536;
10+
SET STATEMENT DEBUG_DBUG="+d,write_to_tmp_file_fail" FOR
11+
CREATE TABLE t2 as SELECT * FROM t1;
12+
ERROR HY000: Got error 59 'Temp file write failure' from InnoDB
13+
DROP TABLE t1;
14+
CREATE TABLE t1(f1 INT NOT NULL, f2 CHAR(100),
15+
f3 CHAR(100))ENGINE=InnoDB;
16+
INSERT INTO t1 SELECT seq, 'a', 'b' FROM seq_1_to_1024;
17+
SET STATEMENT DEBUG_DBUG="+d,write_to_tmp_file_fail" FOR
18+
ALTER TABLE t1 FORCE, ALGORITHM=COPY;
19+
ERROR HY000: Got error 59 'Temp file write failure' from InnoDB
20+
DROP TABLE t1;
21+
CREATE TABLE t1(f1 INT NOT NULL, f2 CHAR(100),
22+
f3 CHAR(100))ENGINE=InnoDB;
23+
INSERT INTO t1 SELECT seq, 'a', 'b' FROM seq_1_to_4096;
24+
SET DEBUG_SYNC="inplace_after_index_build SIGNAL dml_start WAIT_FOR dml_commit";
25+
ALTER TABLE t1 ADD KEY(f1), ADD INDEX(f3(10));
26+
connect con1,localhost,root,,,;
27+
SET DEBUG_SYNC="now WAIT_FOR dml_start";
28+
BEGIN;
29+
INSERT INTO t1 SELECT * FROM t1;
30+
SET STATEMENT DEBUG_DBUG="+d,os_file_write_fail" FOR COMMIT;
31+
SET DEBUG_SYNC="now SIGNAL dml_commit";
32+
connection default;
33+
ERROR HY000: Temporary file write failure
34+
disconnect con1;
35+
CHECK TABLE t1;
36+
Table Op Msg_type Msg_text
37+
test.t1 check status OK
38+
DROP TABLE t1;
39+
SET STATEMENT DEBUG_DBUG="+d,ddl_log_write_fail" FOR
40+
CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB;
41+
DROP TABLE t1;
42+
CREATE TABLE t1(f1 TEXT, index(f1(2)))ENGINE=InnoDB;
43+
INSERT INTO t1 VALUES('a');
44+
set statement DEBUG_DBUG="+d,btr_page_alloc_fail" for
45+
UPDATE t1 set f1= REPEAT('b', 12000);
46+
ERROR HY000: The table 't1' is full
47+
DROP TABLE t1;

‎mysql-test/suite/innodb/t/alloc_fail.opt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
--innodb_sort_buffer_size=64k
2+
--innodb_rollback_on_timeout=1

‎mysql-test/suite/innodb/t/alloc_fail.test‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
--source include/have_innodb.inc
2+
--source include/have_sequence.inc
3+
--source include/have_debug.inc
4+
--echo #
5+
--echo # MDEV-36017 Alter table aborts when temporary
6+
--echo # directory is full
7+
--echo #
8+
SET SESSION DEFAULT_STORAGE_ENGINE=InnoDB;
9+
CREATE TABLE t1(f1 CHAR(100) NOT NULL, f2 CHAR(100) NOT NULL,
10+
f3 CHAR(100) NOT NULL, f4 CHAR(100) NOT NULL,
11+
f5 CHAR(100) NOT NULL)ENGINE=InnoDB;
12+
INSERT INTO t1 SELECT 'a', 'b', 'c', 'd', 'e' FROM seq_1_to_65536;
13+
--error ER_GET_ERRMSG
14+
SET STATEMENT DEBUG_DBUG="+d,write_to_tmp_file_fail" FOR
15+
CREATE TABLE t2 as SELECT * FROM t1;
16+
DROP TABLE t1;
17+
18+
CREATE TABLE t1(f1 INT NOT NULL, f2 CHAR(100),
19+
f3 CHAR(100))ENGINE=InnoDB;
20+
INSERT INTO t1 SELECT seq, 'a', 'b' FROM seq_1_to_1024;
21+
--error ER_GET_ERRMSG
22+
SET STATEMENT DEBUG_DBUG="+d,write_to_tmp_file_fail" FOR
23+
ALTER TABLE t1 FORCE, ALGORITHM=COPY;
24+
DROP TABLE t1;
25+
26+
CREATE TABLE t1(f1 INT NOT NULL, f2 CHAR(100),
27+
f3 CHAR(100))ENGINE=InnoDB;
28+
INSERT INTO t1 SELECT seq, 'a', 'b' FROM seq_1_to_4096;
29+
SET DEBUG_SYNC="inplace_after_index_build SIGNAL dml_start WAIT_FOR dml_commit";
30+
SEND ALTER TABLE t1 ADD KEY(f1), ADD INDEX(f3(10));
31+
32+
connect(con1,localhost,root,,,);
33+
SET DEBUG_SYNC="now WAIT_FOR dml_start";
34+
BEGIN;
35+
INSERT INTO t1 SELECT * FROM t1;
36+
SET STATEMENT DEBUG_DBUG="+d,os_file_write_fail" FOR COMMIT;
37+
SET DEBUG_SYNC="now SIGNAL dml_commit";
38+
39+
connection default;
40+
--error ER_TEMP_FILE_WRITE_FAILURE
41+
reap;
42+
disconnect con1;
43+
CHECK TABLE t1;
44+
DROP TABLE t1;
45+
46+
SET STATEMENT DEBUG_DBUG="+d,ddl_log_write_fail" FOR
47+
CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB;
48+
DROP TABLE t1;
49+
50+
CREATE TABLE t1(f1 TEXT, index(f1(2)))ENGINE=InnoDB;
51+
INSERT INTO t1 VALUES('a');
52+
--error ER_RECORD_FILE_FULL
53+
set statement DEBUG_DBUG="+d,btr_page_alloc_fail" for
54+
UPDATE t1 set f1= REPEAT('b', 12000);
55+
DROP TABLE t1;

‎sql/ddl_log.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3051,13 +3051,15 @@ static bool ddl_log_write(DDL_LOG_STATE *ddl_state,
30513051
error= ((ddl_log_write_entry(ddl_log_entry, &log_entry)) ||
30523052
ddl_log_write_execute_entry(log_entry->entry_pos, 0,
30533053
&ddl_state->execute_entry));
3054-
mysql_mutex_unlock(&LOCK_gdl);
3054+
DBUG_EXECUTE_IF("ddl_log_write_fail", error= true;);
30553055
if (error)
30563056
{
30573057
if (log_entry)
30583058
ddl_log_release_memory_entry(log_entry);
3059+
mysql_mutex_unlock(&LOCK_gdl);
30593060
DBUG_RETURN(1);
30603061
}
3062+
mysql_mutex_unlock(&LOCK_gdl);
30613063
ddl_log_add_entry(ddl_state, log_entry);
30623064
ddl_state->flags|= ddl_log_entry->flags; // Update cache
30633065
DBUG_RETURN(0);

‎storage/innobase/btr/btr0cur.cc‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3697,8 +3697,10 @@ btr_cur_optimistic_update(
36973697
*offsets = rec_get_offsets(rec, index, *offsets, index->n_core_fields,
36983698
ULINT_UNDEFINED, heap);
36993699
#if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG
3700+
/* Blob pointer can be null if InnoDB was killed or
3701+
ran out of space while allocating a page. */
37003702
ut_a(!rec_offs_any_null_extern(rec, *offsets)
3701-
|| thr_get_trx(thr) == trx_roll_crash_recv_trx);
3703+
|| thr_get_trx(thr)->in_rollback);
37023704
#endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */
37033705

37043706
if (UNIV_LIKELY(!update->is_metadata())
@@ -4371,7 +4373,12 @@ btr_cur_pessimistic_update(
43714373
cursor, offsets, offsets_heap,
43724374
new_entry, &rec,
43734375
&dummy_big_rec, n_ext, NULL, mtr);
4374-
ut_a(err == DB_SUCCESS);
4376+
if (err) {
4377+
/* This should happen when InnoDB tries to extend the
4378+
tablespace */
4379+
ut_ad(err == DB_OUT_OF_FILE_SPACE);
4380+
return err;
4381+
}
43754382
ut_a(rec);
43764383
ut_a(dummy_big_rec == NULL);
43774384
ut_ad(rec_offs_validate(rec, cursor->index(), *offsets));
@@ -6240,6 +6247,9 @@ btr_store_big_rec_extern_fields(
62406247
FSP_NO_DIR, 0, &mtr, &mtr,
62416248
&error);
62426249

6250+
DBUG_EXECUTE_IF("btr_page_alloc_fail",
6251+
block= nullptr;
6252+
error= DB_OUT_OF_FILE_SPACE;);
62436253
if (!block) {
62446254
alloc_fail:
62456255
mtr.commit();

‎storage/innobase/fil/fil0fil.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -605,7 +605,7 @@ fil_space_extend_must_retry(
605605
*success = os_file_set_size(node->name, node->handle, new_size,
606606
node->punch_hole == 1);
607607

608-
os_has_said_disk_full = *success;
608+
os_has_said_disk_full = !*success;
609609
if (*success) {
610610
os_file_flush(node->handle);
611611
last_page_no = size;

‎storage/innobase/handler/ha_innodb.cc‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2161,6 +2161,11 @@ convert_error_code_to_mysql(
21612161
return(HA_ERR_RECORD_FILE_FULL);
21622162

21632163
case DB_TEMP_FILE_WRITE_FAIL:
2164+
/* This error can happen during
2165+
copy_data_between_tables() or bulk insert operation */
2166+
innodb_transaction_abort(thd,
2167+
innobase_rollback_on_timeout,
2168+
error);
21642169
my_error(ER_GET_ERRMSG, MYF(0),
21652170
DB_TEMP_FILE_WRITE_FAIL,
21662171
ut_strerr(DB_TEMP_FILE_WRITE_FAIL),
@@ -15931,7 +15936,7 @@ ha_innobase::extra(
1593115936
}
1593215937
m_prebuilt->table->skip_alter_undo = 0;
1593315938
if (dberr_t err= trx->bulk_insert_apply<TRX_DDL_BULK>()) {
15934-
m_prebuilt->table->skip_alter_undo = 0;
15939+
trx->rollback();
1593515940
return convert_error_code_to_mysql(
1593615941
err, m_prebuilt->table->flags,
1593715942
trx->mysql_thd);

‎storage/innobase/row/row0log.cc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,12 +398,17 @@ bool row_log_online_op(dict_index_t *index, const dtuple_t *tuple,
398398
}
399399

400400
log->tail.blocks++;
401+
DBUG_EXECUTE_IF("os_file_write_fail",
402+
log->error = DB_TEMP_FILE_WRITE_FAIL;
403+
goto write_failed;);
404+
401405
if (os_file_write(
402406
IORequestWrite,
403407
"(modification log)",
404408
log->fd,
405409
buf, byte_offset, srv_sort_buf_size)
406410
!= DB_SUCCESS) {
411+
log->error = DB_TEMP_FILE_WRITE_FAIL;
407412
write_failed:
408413
index->type |= DICT_CORRUPT;
409414
}

‎storage/innobase/row/row0merge.cc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5177,6 +5177,9 @@ dberr_t row_merge_bulk_t::write_to_tmp_file(ulint index_no)
51775177
m_block, m_crypt_block,
51785178
buf->index->table->space->id))
51795179
return DB_TEMP_FILE_WRITE_FAIL;
5180+
5181+
DBUG_EXECUTE_IF("write_to_tmp_file_fail",
5182+
return DB_TEMP_FILE_WRITE_FAIL;);
51805183
MEM_UNDEFINED(&m_block[0], srv_sort_buf_size);
51815184
return DB_SUCCESS;
51825185
}

‎storage/innobase/row/row0mysql.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -708,6 +708,7 @@ row_mysql_handle_errors(
708708
case DB_DEADLOCK:
709709
case DB_RECORD_CHANGED:
710710
case DB_LOCK_TABLE_FULL:
711+
case DB_TEMP_FILE_WRITE_FAIL:
711712
rollback:
712713
/* Roll back the whole transaction; this resolution was added
713714
to version 3.23.43 */

0 commit comments

Comments
 (0)