Skip to content

Commit

Permalink
MDEV-15855 Deadlock between purge thread and DDL statement
Browse files Browse the repository at this point in the history 
Problem:
========
Truncate operation holds MDL on the table (t1) and tries to
acquire InnoDB dict_operation_lock. Purge holds dict_operation_lock
and tries to acquire MDL on the table (t1) to evaluate virtual
column expressions for indexed virtual columns.
It leads to deadlock of purge and truncate table (DDL).

Solution:
=========
If purge tries to acquire MDL on the table then it should do the following:

i) Purge should release all innodb latches (including dict_operation_lock)
before acquiring metadata lock on the table.

ii) After acquiring metadata lock on the table, it should check whether the
table was dropped or renamed. If the table is dropped then purge should
ignore the undo log record. If the table is renamed then it should
release the old MDL and acquire MDL on the new name.

iii) Once purge acquires MDL, it should use the SQL table handle for all
the remaining virtual index for the purge record.

purge_node_t: Introduce new virtual column information to know whether
the MDL was acquired successfully.

This is joint work with Marko Mäkelä.
  • Loading branch information
@Thirunarayanan @dr-m
Thirunarayanan authored and dr-m committed Jul 6, 2018
1 parent e3207b6 commit 8b0d4cf
Show file tree
Hide file tree
Showing 14 changed files with 633 additions and 164 deletions.
32 changes: 31 additions & 1 deletion mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result
View file
Expand Up @@ -175,8 +175,38 @@ SET DEBUG_SYNC='now WAIT_FOR halfway';
COMMIT;
InnoDB 0 transactions not purged
SET DEBUG_SYNC='now SIGNAL purged';
disconnect prevent_purge;
connection default;
DROP TABLE t1;
CREATE TABLE t1 (y YEAR, vy YEAR AS (y) VIRTUAL UNIQUE, pk INT PRIMARY KEY)
ENGINE=InnoDB;
INSERT INTO t1 (pk,y) VALUES (1,2022);
CREATE TABLE t2(f1 INT NOT NULL, PRIMARY KEY(f1))ENGINE=InnoDB;
SET GLOBAL debug_dbug = '+d,ib_purge_virtual_index_callback';
BEGIN;
INSERT INTO t2(f1) VALUES(1);
connection prevent_purge;
SET DEBUG_SYNC=RESET;
start transaction with consistent snapshot;
connection default;
COMMIT;
connect truncate,localhost,root,,;
REPLACE INTO t1(pk, y) SELECT pk,y FROM t1;
SET DEBUG_SYNC='row_trunc_before_dict_lock SIGNAL commit WAIT_FOR release';
TRUNCATE TABLE t1;
connection prevent_purge;
SET DEBUG_SYNC='now WAIT_FOR commit';
COMMIT;
SET DEBUG_SYNC='now SIGNAL purge_start';
disconnect prevent_purge;
connection default;
SET DEBUG_SYNC='now WAIT_FOR purge_start';
InnoDB 1 transactions not purged
SET DEBUG_SYNC='now SIGNAL release';
SET GLOBAL debug_dbug=@old_dbug;
connection truncate;
disconnect truncate;
connection default;
InnoDB 0 transactions not purged
DROP TABLE t1, t2;
set debug_sync=reset;
SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
44 changes: 43 additions & 1 deletion mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test
View file
Expand Up @@ -217,12 +217,54 @@ SET DEBUG_SYNC='now WAIT_FOR halfway';
COMMIT;
--source ../../innodb/include/wait_all_purged.inc
SET DEBUG_SYNC='now SIGNAL purged';
disconnect prevent_purge;

connection default;
reap;
DROP TABLE t1;

CREATE TABLE t1 (y YEAR, vy YEAR AS (y) VIRTUAL UNIQUE, pk INT PRIMARY KEY)
ENGINE=InnoDB;

INSERT INTO t1 (pk,y) VALUES (1,2022);
CREATE TABLE t2(f1 INT NOT NULL, PRIMARY KEY(f1))ENGINE=InnoDB;

SET GLOBAL debug_dbug = '+d,ib_purge_virtual_index_callback';

BEGIN;
INSERT INTO t2(f1) VALUES(1);
connection prevent_purge;
SET DEBUG_SYNC=RESET;
start transaction with consistent snapshot;
connection default;
COMMIT;

connect(truncate,localhost,root,,);
REPLACE INTO t1(pk, y) SELECT pk,y FROM t1;
SET DEBUG_SYNC='row_trunc_before_dict_lock SIGNAL commit WAIT_FOR release';
send TRUNCATE TABLE t1;

connection prevent_purge;
SET DEBUG_SYNC='now WAIT_FOR commit';
COMMIT;
SET DEBUG_SYNC='now SIGNAL purge_start';
disconnect prevent_purge;

connection default;
SET DEBUG_SYNC='now WAIT_FOR purge_start';
let $wait_all_purged=1;
--source ../../innodb/include/wait_all_purged.inc
let $wait_all_purged=0;
SET DEBUG_SYNC='now SIGNAL release';
SET GLOBAL debug_dbug=@old_dbug;

connection truncate;
reap;
disconnect truncate;

connection default;
--source ../../innodb/include/wait_all_purged.inc
DROP TABLE t1, t2;

--source include/wait_until_count_sessions.inc
set debug_sync=reset;
SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
7 changes: 0 additions & 7 deletions sql/sql_class.cc
View file
Expand Up @@ -4404,13 +4404,6 @@ TABLE *open_purge_table(THD *thd, const char *db, size_t dblen,
DBUG_RETURN(error ? NULL : tl->table);
}

TABLE *get_purge_table(THD *thd)
{
/* see above, at most one table can be opened */
DBUG_ASSERT(thd->open_tables == NULL || thd->open_tables->next == NULL);
return thd->open_tables;
}


/** Find an open table in the list of prelocked tabled
Expand Down
203 changes: 150 additions & 53 deletions storage/innobase/handler/ha_innodb.cc
View file
Expand Up @@ -129,7 +129,7 @@ void destroy_thd(MYSQL_THD thd);
void reset_thd(MYSQL_THD thd);
TABLE *open_purge_table(THD *thd, const char *db, size_t dblen,
const char *tb, size_t tblen);
TABLE *get_purge_table(THD *thd);
void close_thread_tables(THD* thd);

/** Check if user has used xtradb extended system variable that
is not currently supported by innodb or marked as deprecated. */
Expand Down Expand Up @@ -21603,63 +21603,154 @@ innobase_index_cond(
return handler_index_cond_check(file);
}

/** Parse the table file name into table name and database name.
@param[in] tbl_name InnoDB table name
@param[out] dbname database name buffer (NAME_LEN + 1 bytes)
@param[out] tblname table name buffer (NAME_LEN + 1 bytes)
@param[out] dbnamelen database name length
@param[out] tblnamelen table name length
@return true if the table name is parsed properly. */
static bool table_name_parse(
const table_name_t& tbl_name,
char* dbname,
char* tblname,
ulint& dbnamelen,
ulint& tblnamelen)
{
dbnamelen = dict_get_db_name_len(tbl_name.m_name);
char db_buf[MAX_DATABASE_NAME_LEN + 1];
char tbl_buf[MAX_TABLE_NAME_LEN + 1];

/** Find or open a mysql table for the virtual column template
@param[in] thd mysql thread handle
@param[in,out] table InnoDB table whose virtual column template is to be updated
@return TABLE if successful or NULL */
static TABLE *
innobase_find_mysql_table_for_vc(
/*=============================*/
THD* thd,
dict_table_t* table)
ut_ad(dbnamelen > 0);
ut_ad(dbnamelen <= MAX_DATABASE_NAME_LEN);

memcpy(db_buf, tbl_name.m_name, dbnamelen);
db_buf[dbnamelen] = 0;

tblnamelen = strlen(tbl_name.m_name + dbnamelen + 1);
memcpy(tbl_buf, tbl_name.m_name + dbnamelen + 1, tblnamelen);
tbl_buf[tblnamelen] = 0;

filename_to_tablename(db_buf, dbname, MAX_DATABASE_NAME_LEN + 1, true);

if (tblnamelen > TEMP_FILE_PREFIX_LENGTH
&& !strncmp(tbl_buf, TEMP_FILE_PREFIX, TEMP_FILE_PREFIX_LENGTH)) {
return false;
}

if (char *is_part = strchr(tbl_buf, '#')) {
*is_part = '\0';
}

filename_to_tablename(tbl_buf, tblname, MAX_TABLE_NAME_LEN + 1, true);
return true;
}


/** Acquire metadata lock and MariaDB table handle for an InnoDB table.
@param[in,out] thd thread handle
@param[in,out] table InnoDB table
@return MariaDB table handle
@retval NULL if the table does not exist, is unaccessible or corrupted. */
static TABLE* innodb_acquire_mdl(THD* thd, dict_table_t* table)
{
TABLE *mysql_table;
bool bg_thread = THDVAR(thd, background_thread);
char db_buf[NAME_LEN + 1], db_buf1[NAME_LEN + 1];
char tbl_buf[NAME_LEN + 1], tbl_buf1[NAME_LEN + 1];
ulint db_buf_len, db_buf1_len;
ulint tbl_buf_len, tbl_buf1_len;

if (bg_thread) {
if ((mysql_table = get_purge_table(thd))) {
return mysql_table;
}
} else {
if (table->vc_templ->mysql_table_query_id == thd_get_query_id(thd)) {
return table->vc_templ->mysql_table;
if (!table_name_parse(table->name, db_buf, tbl_buf,
db_buf_len, tbl_buf_len)) {
ut_ad(!"invalid table name");
return NULL;
}

const table_id_t table_id = table->id;
retry_mdl:
const bool unaccessible = !table->is_readable() || table->corrupted;
table->release();

if (unaccessible) {
return NULL;
}

TABLE* mariadb_table = open_purge_table(thd, db_buf, db_buf_len,
tbl_buf, tbl_buf_len);

table = dict_table_open_on_id(table_id, false, DICT_TABLE_OP_NORMAL);

if (table == NULL) {
/* Table is dropped. */
goto fail;
}

if (!fil_table_accessible(table)) {
release_fail:
table->release();
fail:
if (mariadb_table) {
close_thread_tables(thd);
}

return NULL;
}

char dbname[MAX_DATABASE_NAME_LEN + 1];
char tbname[MAX_TABLE_NAME_LEN + 1];
char* name = table->name.m_name;
uint dbnamelen = (uint) dict_get_db_name_len(name);
uint tbnamelen = (uint) strlen(name) - dbnamelen - 1;
char t_dbname[MAX_DATABASE_NAME_LEN + 1];
char t_tbname[MAX_TABLE_NAME_LEN + 1];
if (!table_name_parse(table->name, db_buf1, tbl_buf1,
db_buf1_len, tbl_buf1_len)) {
ut_ad(!"invalid table name");
goto release_fail;
}

strncpy(dbname, name, dbnamelen);
dbname[dbnamelen] = 0;
strncpy(tbname, name + dbnamelen + 1, tbnamelen);
tbname[tbnamelen] =0;
if (!mariadb_table) {
} else if (!strcmp(db_buf, db_buf1) && !strcmp(tbl_buf, tbl_buf1)) {
return mariadb_table;
} else {
/* Table is renamed. So release MDL for old name and try
to acquire the MDL for new table name. */
close_thread_tables(thd);
}

/* For partition table, remove the partition name and use the
"main" table name to build the template */
char* is_part = is_partition(tbname);
strcpy(tbl_buf, tbl_buf1);
strcpy(db_buf, db_buf1);
tbl_buf_len = tbl_buf1_len;
db_buf_len = db_buf1_len;
goto retry_mdl;
}

if (is_part != NULL) {
*is_part = '\0';
/** Find or open a table handle for the virtual column template
@param[in] thd thread handle
@param[in,out] table InnoDB table whose virtual column template
is to be updated
@return table handle
@retval NULL if the table is dropped, unaccessible or corrupted
for purge thread */
static TABLE* innodb_find_table_for_vc(THD* thd, dict_table_t* table)
{
if (THDVAR(thd, background_thread)) {
/* Purge thread acquires dict_operation_lock while
processing undo log record. Release the dict_operation_lock
before acquiring MDL on the table. */
rw_lock_s_unlock(dict_operation_lock);
return innodb_acquire_mdl(thd, table);
} else {
if (table->vc_templ->mysql_table_query_id
== thd_get_query_id(thd)) {
return table->vc_templ->mysql_table;
}
}

dbnamelen = filename_to_tablename(dbname, t_dbname,
MAX_DATABASE_NAME_LEN + 1);
tbnamelen = filename_to_tablename(tbname, t_tbname,
MAX_TABLE_NAME_LEN + 1);
char db_buf[NAME_LEN + 1];
char tbl_buf[NAME_LEN + 1];
ulint db_buf_len, tbl_buf_len;

if (bg_thread) {
return open_purge_table(thd, t_dbname, dbnamelen,
t_tbname, tbnamelen);
if (!table_name_parse(table->name, db_buf, tbl_buf,
db_buf_len, tbl_buf_len)) {
ut_ad(!"invalid table name");
return NULL;
}

mysql_table = find_fk_open_table(thd, t_dbname, dbnamelen,
t_tbname, tbnamelen);
TABLE* mysql_table = find_fk_open_table(thd, db_buf, db_buf_len,
tbl_buf, tbl_buf_len);

table->vc_templ->mysql_table = mysql_table;
table->vc_templ->mysql_table_query_id = thd_get_query_id(thd);
Expand All @@ -21678,7 +21769,7 @@ innobase_init_vc_templ(

table->vc_templ = UT_NEW_NOKEY(dict_vcol_templ_t());

TABLE *mysql_table= innobase_find_mysql_table_for_vc(current_thd, table);
TABLE *mysql_table= innodb_find_table_for_vc(current_thd, table);

ut_ad(mysql_table);
if (!mysql_table) {
Expand Down Expand Up @@ -21772,15 +21863,16 @@ innobase_get_field_from_update_vector(
Allocate a heap and record for calculating virtual fields
Used mainly for virtual fields in indexes

@param[in] thd MariaDB THD
@param[in] index Index in use
@param[in] thd MariaDB THD
@param[in] index Index in use
@param[out] heap Heap that holds temporary row
@param[in,out] mysql_table MariaDB table
@param[out] rec Pointer to allocated MariaDB record
@param[out] storage Internal storage for blobs etc
@param[in,out] table MariaDB table
@param[out] record Pointer to allocated MariaDB record
@param[out] storage Internal storage for blobs etc

@return FALSE ok
@return TRUE malloc failure
@retval false on success
@retval true on malloc failure or failed to open the maria table
for purge thread.
*/

bool innobase_allocate_row_for_vcol(
Expand All @@ -21794,7 +21886,12 @@ bool innobase_allocate_row_for_vcol(
TABLE *maria_table;
String *blob_value_storage;
if (!*table)
*table= innobase_find_mysql_table_for_vc(thd, index->table);
*table= innodb_find_table_for_vc(thd, index->table);

/* For purge thread, there is a possiblity that table could have
dropped, corrupted or unaccessible. */
if (!*table)
return true;
maria_table= *table;
if (!*heap && !(*heap= mem_heap_create(srv_page_size)))
{
Expand Down
14 changes: 9 additions & 5 deletions storage/innobase/include/btr0btr.h
View file
Expand Up @@ -115,7 +115,15 @@ enum btr_latch_mode {
/** Attempt to purge a secondary index record
while holding the dict_index_t::lock S-latch. */
BTR_PURGE_LEAF_ALREADY_S_LATCHED = BTR_PURGE_LEAF
| BTR_ALREADY_S_LATCHED
| BTR_ALREADY_S_LATCHED,

/** In the case of BTR_MODIFY_TREE, the caller specifies
the intention to delete record only. It is used to optimize
block->lock range.*/
BTR_LATCH_FOR_DELETE = 65536,

/** Attempt to purge a secondary index record in the tree. */
BTR_PURGE_TREE = BTR_MODIFY_TREE | BTR_LATCH_FOR_DELETE
};

/** This flag ORed to btr_latch_mode says that we do the search in query
Expand All @@ -131,10 +139,6 @@ the insert buffer to speed up inserts */
to insert record only. It is used to optimize block->lock range.*/
#define BTR_LATCH_FOR_INSERT 32768U

/** In the case of BTR_MODIFY_TREE, the caller specifies the intention
to delete record only. It is used to optimize block->lock range.*/
#define BTR_LATCH_FOR_DELETE 65536U

/** This flag is for undo insert of rtree. For rtree, we need this flag
to find proper rec to undo insert.*/
#define BTR_RTREE_UNDO_INS 131072U
Expand Down

0 comments on commit 8b0d4cf

Please sign in to comment.