Bug#34929814 Inconsistent FTS state in concurrent scenarios

Bug#36347647  Contribution by Tencent: Resiliency issue in fts_sync_commit

Symptoms:
During various operations on tables containing FTS indexes
the state of FTS as comitted to database may become
inconsistent, affecting the following scenarios:
 - the server terminates when synchronizing the FTS cache,
 - synchronization of FTS cache occurs concurrently with
   another FTS operation
This inconsistency may lead to various negative effects
including incorrect query results.
An example operation which forces the synchronization of
FTS cach is OPTIMIZE TABLE with
innodb_optimize_fulltext_only set to ON.

Root Cause:
Function 'fts_cmp_set_sync_doc_id' and 'fts_sql_commit' use
different  trx_t objects in function 'fts_sync_commit'.
This causes a scenario where 'synced_doc_id' in the config
table is already committed, but remaining FTS data isn't yet,
leading to issues in the scenarios described above - the
server terminating between the commits, or concurrent access
getting the intermediate state.

Fix:
When 'fts_cmp_set_sync_doc_id' is called from 'fts_sync_commit'
it will use the transaction provided by the caller.

Patch based on contribution by Tencent.

Change-Id: I65fa5702db5e7b6b2004a7311a6b0aa97449034f

mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result

@@ -0,0 +1,63 @@
+CREATE TABLE opening_lines (
+id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
+opening_line TEXT(500),
+author VARCHAR(200),
+title VARCHAR(200)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx ON opening_lines(opening_line);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title);
+INSERT INTO opening_lines(opening_line,author,title) VALUES
+('Call me Ishmael.','Herman Melville','Moby Dick'),
+('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'),
+       ('I am an invisible man.','Ralph Ellison','Invisible Man'),
+       ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'),
+       ('It was love at first sight.','Joseph Heller','Catch-22'),
+       ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'),
+       ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'),
+       ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451');
+SET GLOBAL innodb_ft_aux_table='test/opening_lines';
+SELECT * FROM information_schema.innodb_ft_config;
+KEY	VALUE
+optimize_checkpoint_limit	180
+synced_doc_id	0
+stopword_table_name	
+use_stopword	1
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+id	opening_line	author	title
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+SELECT * FROM opening_lines;
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+2	A screaming comes across the sky.	Thomas Pynchon	Gravity's Rainbow
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+4	Where now? Who now? When now?	Samuel Beckett	The Unnamable
+5	It was love at first sight.	Joseph Heller	Catch-22
+6	All this happened, more or less.	Kurt Vonnegut	Slaughterhouse-Five
+7	Mrs. Dalloway said she would buy the flowers herself.	Virginia Woolf	Mrs. Dalloway
+8	It was a pleasure to burn.	Ray Bradbury	Fahrenheit 451
+SET GLOBAL innodb_optimize_fulltext_only=ON;
+SET GLOBAL debug='+d,fts_crash_before_commit_sync';
+OPTIMIZE TABLE opening_lines;
+ERROR HY000: Lost connection to MySQL server during query
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+id	opening_line	author	title
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+SELECT * FROM opening_lines;
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+2	A screaming comes across the sky.	Thomas Pynchon	Gravity's Rainbow
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+4	Where now? Who now? When now?	Samuel Beckett	The Unnamable
+5	It was love at first sight.	Joseph Heller	Catch-22
+6	All this happened, more or less.	Kurt Vonnegut	Slaughterhouse-Five
+7	Mrs. Dalloway said she would buy the flowers herself.	Virginia Woolf	Mrs. Dalloway
+8	It was a pleasure to burn.	Ray Bradbury	Fahrenheit 451
+DROP TABLE opening_lines;

mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test

@@ -0,0 +1,46 @@
+# Test database resiliency against scenario where the server crashes
+# right before fts_sync_commit commits its transaction
+
+source include/have_debug.inc;
+source include/not_valgrind.inc;
+
+CREATE TABLE opening_lines (
+       id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
+       opening_line TEXT(500),
+       author VARCHAR(200),
+       title VARCHAR(200)
+       ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx ON opening_lines(opening_line);
+CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title);
+
+INSERT INTO opening_lines(opening_line,author,title) VALUES
+       ('Call me Ishmael.','Herman Melville','Moby Dick'),
+       ('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'),
+       ('I am an invisible man.','Ralph Ellison','Invisible Man'),
+       ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'),
+       ('It was love at first sight.','Joseph Heller','Catch-22'),
+       ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'),
+       ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'),
+       ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451');
+
+SET GLOBAL innodb_ft_aux_table='test/opening_lines';
+SELECT * FROM information_schema.innodb_ft_config;
+
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+SELECT * FROM opening_lines;
+
+SET GLOBAL innodb_optimize_fulltext_only=ON;
+SET GLOBAL debug='+d,fts_crash_before_commit_sync';
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--error CR_SERVER_LOST
+OPTIMIZE TABLE opening_lines;
+
+--source include/wait_until_connected_again.inc
+
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+SELECT * FROM opening_lines;
+
+DROP TABLE opening_lines;

storage/innobase/fts/fts0fts.cc

@@ -2795,18 +2795,19 @@ static dberr_t fts_cmp_set_sync_doc_id(
     doc_id_t doc_id_cmp,       /*!< in: Doc ID to compare */
     bool read_only,            /*!< in: true if read the
                                 synced_doc_id only */
-    doc_id_t *doc_id)          /*!< out: larger document id
+    doc_id_t *doc_id,          /*!< out: larger document id
                                after comparing "doc_id_cmp"
                                to the one stored in CONFIG
                                table */
-{
-  trx_t *trx;
+    trx_t *trx = nullptr) {
   pars_info_t *info;
   dberr_t error;
   fts_table_t fts_table;
   que_t *graph = nullptr;
   fts_cache_t *cache = table->fts->cache;
   char table_name[MAX_FULL_NAME_LEN];
+  bool trx_allocated;
+  trx_savept_t savept;
 retry:
   ut_a(table->fts->doc_col != ULINT_UNDEFINED);
 
@@ -2817,7 +2818,13 @@ static dberr_t fts_cmp_set_sync_doc_id(
 
   fts_table.parent = table->name.m_name;
 
-  trx = trx_allocate_for_background();
+  trx_allocated = false;
+  if (trx == nullptr) {
+    trx = trx_allocate_for_background();
+    trx_allocated = true;
+  } else {
+    savept = trx_savept_take(trx);
+  }
 
   trx->op_info = "update the next FTS document id";
 
@@ -2882,23 +2889,36 @@ static dberr_t fts_cmp_set_sync_doc_id(
 func_exit:
 
   if (error == DB_SUCCESS) {
-    fts_sql_commit(trx);
+    if (trx_allocated) {
+      fts_sql_commit(trx);
+    }
   } else {
     *doc_id = 0;
 
     ib::error(ER_IB_MSG_471) << "(" << ut_strerr(error)
                              << ") while getting"
                                 " next doc id.";
-    fts_sql_rollback(trx);
+    if (trx_allocated) {
+      fts_sql_rollback(trx);
+    } else {
+      trx_rollback_to_savepoint(trx, &savept);
+    }
 
     if (error == DB_DEADLOCK) {
       std::this_thread::sleep_for(
           std::chrono::milliseconds(FTS_DEADLOCK_RETRY_WAIT_MS));
+      if (trx_allocated) {
+        /* free trx before retry */
+        trx_free_for_background(trx);
+        trx = nullptr;
+      }
       goto retry;
     }
   }
 
-  trx_free_for_background(trx);
+  if (trx_allocated) {
+    trx_free_for_background(trx);
+  }
 
   return (error);
 }
@@ -4251,7 +4271,7 @@ static void fts_sync_index_reset(fts_index_cache_t *index_cache) {
   /* After each Sync, update the CONFIG table about the max doc id
   we just sync-ed to index table */
   error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, false,
-                                  &last_doc_id);
+                                  &last_doc_id, trx);
 
   /* Get the list of deleted documents that are either in the
   cache or were headed there but were deleted before the add
@@ -4269,6 +4289,7 @@ static void fts_sync_index_reset(fts_index_cache_t *index_cache) {
   rw_lock_x_unlock(&cache->lock);
 
   if (error == DB_SUCCESS) {
+    DBUG_EXECUTE_IF("fts_crash_before_commit_sync", { DBUG_SUICIDE(); });
     fts_sql_commit(trx);
 
   } else if (error != DB_SUCCESS) {