5.6.38-83.0

storage/xtradb/btr/btr0pcur.cc

@@ -33,6 +33,40 @@ Created 2/23/1996 Heikki Tuuri
 #include "rem0cmp.h"
 #include "trx0trx.h"
 #include "srv0srv.h"
+
+/** Updates fragmentation statistics for a single page transition.
+@param[in]	page			the current page being processed
+@param[in]	page_no			page number to move to (next_page_no
+					if forward_direction is true,
+					prev_page_no otherwise.
+@param[in]	forward_direction	move direction: true means moving
+					forward, false - backward. */
+static
+void
+btr_update_scan_stats(const page_t* page, ulint page_no, bool forward_direction)
+{
+	fragmentation_stats_t	stats;
+	memset(&stats, 0, sizeof(stats));
+	ulint	extracted_page_no = page_get_page_no(page);
+	ulint	delta = forward_direction ?
+		page_no - extracted_page_no :
+		extracted_page_no - page_no;
+
+	if (delta == 1) {
+		++stats.scan_pages_contiguous;
+	} else {
+		++stats.scan_pages_disjointed;
+	}
+	stats.scan_pages_total_seek_distance +=
+		extracted_page_no > page_no ?
+		extracted_page_no - page_no :
+		page_no - extracted_page_no;
+
+	stats.scan_data_size += page_get_data_size(page);
+	stats.scan_deleted_recs_size +=
+		page_header_get_field(page, PAGE_GARBAGE);
+	thd_add_fragmentation_stats(current_thd, &stats);
+}
 /**************************************************************//**
 Allocates memory for a persistent cursor object and initializes the cursor.
 @return	own: persistent cursor */
@@ -426,6 +460,8 @@ btr_pcur_move_to_next_page(
 
 	ut_ad(next_page_no != FIL_NULL);
 
+	btr_update_scan_stats(page, next_page_no, true /* forward */);
+
 	next_block = btr_block_get(space, zip_size, next_page_no,
 				   cursor->latch_mode,
 				   btr_pcur_get_btr_cur(cursor)->index, mtr);
@@ -509,6 +545,10 @@ btr_pcur_move_backward_from_page(
 
 	prev_page_no = btr_page_get_prev(page, mtr);
 
+	if (prev_page_no != FIL_NULL) {
+		btr_update_scan_stats(page, prev_page_no, false /* backward */);
+	}
+
 	if (prev_page_no == FIL_NULL) {
 	} else if (btr_pcur_is_before_first_on_page(cursor)) {
 

storage/xtradb/buf/buf0rea.cc

@@ -123,7 +123,12 @@ buf_read_page_low(
 			use to stop dangling page reads from a tablespace
 			which we have DISCARDed + IMPORTed back */
 	ulint	offset,	/*!< in: page number */
-	trx_t*	trx)
+	trx_t*	trx,
+	bool	should_buffer)	/*!< in: whether to buffer an aio request.
+			AIO read ahead uses this. If you plan to
+			use this parameter, make sure you remember
+			to call os_aio_dispatch_read_array_submit()
+			when you're ready to commit all your requests.*/
 {
 	buf_page_t*	bpage;
 	ulint		wake_later;
@@ -229,14 +234,15 @@ buf_read_page_low(
 		*err = _fil_io(OS_FILE_READ | wake_later
 			       | ignore_nonexistent_pages,
 			       sync, space, zip_size, offset, 0, zip_size,
-			       bpage->zip.data, bpage, trx);
+			       bpage->zip.data, bpage, trx, should_buffer);
 	} else {
 		ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE);
 
 		*err = _fil_io(OS_FILE_READ | wake_later
 			      | ignore_nonexistent_pages,
 			      sync, space, 0, offset, 0, UNIV_PAGE_SIZE,
-			      ((buf_block_t*) bpage)->frame, bpage, trx);
+			      ((buf_block_t*) bpage)->frame, bpage, trx,
+			      should_buffer);
 	}
 
 	if (sync) {
@@ -395,7 +401,7 @@ buf_read_ahead_random(
 				&err, false,
 				ibuf_mode | OS_AIO_SIMULATED_WAKE_LATER,
 				space, zip_size, FALSE,
-				tablespace_version, i, trx);
+				tablespace_version, i, trx, false);
 			if (err == DB_TABLESPACE_DELETED) {
 				ut_print_timestamp(stderr);
 				fprintf(stderr,
@@ -459,7 +465,7 @@ buf_read_page(
 
 	count = buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space,
 				  zip_size, FALSE,
-				  tablespace_version, offset, trx);
+				  tablespace_version, offset, trx, false);
 	srv_stats.buf_pool_reads.add(count);
 	if (err == DB_TABLESPACE_DELETED) {
 		ut_print_timestamp(stderr);
@@ -507,7 +513,7 @@ buf_read_page_async(
 				  | OS_AIO_SIMULATED_WAKE_LATER
 				  | BUF_READ_IGNORE_NONEXISTENT_PAGES,
 				  space, zip_size, FALSE,
-				  tablespace_version, offset, NULL);
+				  tablespace_version, offset, NULL, false);
 	srv_stats.buf_pool_reads.add(count);
 
 	/* We do not increment number of I/O operations used for LRU policy
@@ -775,7 +781,8 @@ buf_read_ahead_linear(
 			count += buf_read_page_low(
 				&err, false,
 				ibuf_mode,
-				space, zip_size, FALSE, tablespace_version, i, trx);
+				space, zip_size, FALSE, tablespace_version,
+				i, trx, true);
 			if (err == DB_TABLESPACE_DELETED) {
 				ut_print_timestamp(stderr);
 				fprintf(stderr,
@@ -788,6 +795,7 @@ buf_read_ahead_linear(
 			}
 		}
 	}
+	os_aio_dispatch_read_array_submit();
 
 	/* In simulated aio we wake the aio handler threads only after
 	queuing all aio requests, in native aio the following call does
@@ -865,7 +873,7 @@ buf_read_ibuf_merge_pages(
 		buf_read_page_low(&err, sync && (i + 1 == n_stored),
 				  BUF_READ_ANY_PAGE, space_ids[i],
 				  zip_size, TRUE, space_versions[i],
-				  page_nos[i], NULL);
+				  page_nos[i], NULL, false);
 
 		if (UNIV_UNLIKELY(err == DB_TABLESPACE_DELETED)) {
 tablespace_deleted:
@@ -1005,12 +1013,13 @@ buf_read_recv_pages(
 		if ((i + 1 == n_stored) && sync) {
 			buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space,
 					  zip_size, TRUE, tablespace_version,
-					  page_nos[i], NULL);
+					  page_nos[i], NULL, false);
 		} else {
 			buf_read_page_low(&err, false, BUF_READ_ANY_PAGE
 					  | OS_AIO_SIMULATED_WAKE_LATER,
 					  space, zip_size, TRUE,
-					  tablespace_version, page_nos[i], NULL);
+					  tablespace_version, page_nos[i],
+					  NULL, false);
 		}
 	}
 

storage/xtradb/dict/dict0stats_bg.cc

@@ -195,7 +195,7 @@ dict_stats_wait_bg_to_stop_using_table(
 				unlocking/locking the data dict */
 {
 	while (!dict_stats_stop_bg(table)) {
-		DICT_STATS_BG_YIELD(trx);
+		DICT_BG_YIELD(trx);
 	}
 }
 

storage/xtradb/fil/fil0fil.cc

@@ -5253,7 +5253,7 @@ fil_extend_space_to_desired_size(
 		success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC,
 				 node->name, node->handle, buf,
 				 offset, page_size * n_pages,
-				 NULL, NULL, space_id, NULL);
+				 NULL, NULL, space_id, NULL, false);
 #endif /* UNIV_HOTBACKUP */
 		if (success) {
 			os_has_said_disk_full = FALSE;
@@ -5630,7 +5630,12 @@ _fil_io(
 				appropriately aligned */
 	void*	message,	/*!< in: message for aio handler if non-sync
 				aio used, else ignored */
-	trx_t*	trx)
+	trx_t*	trx,
+	bool	should_buffer)	/*!< in: whether to buffer an aio request.
+				AIO read ahead uses this. If you plan to
+				use this parameter, make sure you remember
+				to call os_aio_dispatch_read_array_submit()
+				when you're ready to commit all your requests.*/
 {
 	ulint		mode;
 	fil_space_t*	space;
@@ -5848,7 +5853,7 @@ _fil_io(
 
 	/* Queue the aio request */
 	ret = os_aio(type, mode | wake_later, node->name, node->handle, buf,
-		offset, len, node, message, space_id, trx);
+		offset, len, node, message, space_id, trx, should_buffer);
 
 #else
 	/* In mysqlbackup do normal i/o, not aio */

storage/xtradb/fts/fts0fts.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -25,6 +25,7 @@ Full Text Search interface
 #include "row0mysql.h"
 #include "row0upd.h"
 #include "dict0types.h"
+#include "dict0stats_bg.h"
 #include "row0sel.h"
 
 #include "fts0fts.h"
@@ -867,18 +868,37 @@ fts_drop_index(
 
 			err = fts_drop_index_tables(trx, index);
 
-			fts_free(table);
-
+			for(;;) {
+				bool retry = false;
+				if (index->index_fts_syncing) {
+					retry = true;
+				}
+				if (!retry){
+					fts_free(table);
+					break;
+				}
+				DICT_BG_YIELD(trx);
+			}
 			return(err);
 		}
 
-		current_doc_id = table->fts->cache->next_doc_id;
-		first_doc_id = table->fts->cache->first_doc_id;
-		fts_cache_clear(table->fts->cache);
-		fts_cache_destroy(table->fts->cache);
-		table->fts->cache = fts_cache_create(table);
-		table->fts->cache->next_doc_id = current_doc_id;
-		table->fts->cache->first_doc_id = first_doc_id;
+		for(;;) {
+			bool retry = false;
+			if (index->index_fts_syncing) {
+				retry = true;
+			}
+			if (!retry){
+				current_doc_id = table->fts->cache->next_doc_id;
+				first_doc_id = table->fts->cache->first_doc_id;
+				fts_cache_clear(table->fts->cache);
+				fts_cache_destroy(table->fts->cache);
+				table->fts->cache = fts_cache_create(table);
+				table->fts->cache->next_doc_id = current_doc_id;
+				table->fts->cache->first_doc_id = first_doc_id;
+				break;
+			}
+			DICT_BG_YIELD(trx);
+		}
 	} else {
 		fts_cache_t*            cache = table->fts->cache;
 		fts_index_cache_t*      index_cache;
@@ -888,9 +908,17 @@ fts_drop_index(
 		index_cache = fts_find_index_cache(cache, index);
 
 		if (index_cache != NULL) {
-			if (index_cache->words) {
-				fts_words_free(index_cache->words);
-				rbt_free(index_cache->words);
+			for(;;) {
+				bool retry = false;
+				if (index->index_fts_syncing) {
+					retry = true;
+				}
+				if (!retry && index_cache->words) {
+					fts_words_free(index_cache->words);
+					rbt_free(index_cache->words);
+					break;
+				}
+				DICT_BG_YIELD(trx);
 			}
 
 			ib_vector_remove(cache->indexes, *(void**) index_cache);
@@ -4611,10 +4639,16 @@ fts_sync(
 		index_cache = static_cast<fts_index_cache_t*>(
 			ib_vector_get(cache->indexes, i));
 
-		if (index_cache->index->to_be_dropped) {
+		if (index_cache->index->to_be_dropped
+		   || index_cache->index->table->to_be_dropped) {
 			continue;
 		}
 
+		index_cache->index->index_fts_syncing = true;
+		DBUG_EXECUTE_IF("fts_instrument_sync_sleep_drop_waits",
+				os_thread_sleep(10000000);
+				);
+
 		error = fts_sync_index(sync, index_cache);
 
 		if (error != DB_SUCCESS && !sync->interrupted) {
@@ -4647,11 +4681,33 @@ fts_sync(
 end_sync:
 	if (error == DB_SUCCESS && !sync->interrupted) {
 		error = fts_sync_commit(sync);
+		if (error == DB_SUCCESS) {
+			for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
+				fts_index_cache_t*      index_cache;
+				index_cache = static_cast<fts_index_cache_t*>(
+					ib_vector_get(cache->indexes, i));
+				if (index_cache->index->index_fts_syncing) {
+					index_cache->index->index_fts_syncing
+								= false;
+				}
+			}
+		}
 	}  else {
 		fts_sync_rollback(sync);
 	}
 
 	rw_lock_x_lock(&cache->lock);
+	/* Clear fts syncing flags of any indexes incase sync is
+	interrupeted */
+	for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
+		fts_index_cache_t*      index_cache;
+		index_cache = static_cast<fts_index_cache_t*>(
+                      ib_vector_get(cache->indexes, i));
+		if (index_cache->index->index_fts_syncing == true) {
+			index_cache->index->index_fts_syncing = false;
+                  }
+	}
+
 	sync->interrupted = false;
 	sync->in_progress = false;
 	os_event_set(sync->event);

storage/xtradb/fts/fts0opt.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2970,13 +2970,6 @@ fts_optimize_sync_table(
 {
 	dict_table_t*   table = NULL;
 
-	/* Prevent DROP INDEX etc. from running when we are syncing
-	cache in background. */
-	if (!rw_lock_s_lock_nowait(&dict_operation_lock, __FILE__, __LINE__)) {
-		/* Exit when fail to get dict operation lock. */
-		return;
-	}
-
 	table = dict_table_open_on_id(table_id, FALSE, DICT_TABLE_OP_NORMAL);
 
 	if (table) {
@@ -2986,8 +2979,6 @@ fts_optimize_sync_table(
 
 		dict_table_close(table, FALSE, FALSE);
 	}
-
-	rw_lock_s_unlock(&dict_operation_lock);
 }
 
 /**********************************************************************//**

storage/xtradb/fts/fts0que.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -3652,6 +3652,11 @@ fts_query_free(
 		fts_doc_ids_free(query->deleted);
 	}
 
+	if (query->intersection) {
+		fts_query_free_doc_ids(query, query->intersection);
+		query->intersection = NULL;
+	}
+
 	if (query->doc_ids) {
 		fts_query_free_doc_ids(query, query->doc_ids);
 	}