MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys

Jan Lindström · Jan Lindström · commit ddaddf1019f0 · 2015-09-14T12:15:27.000+03:00
Add error handling when getting block from encrypted table and decryption
fails.
diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change5.result b/mysql-test/suite/encryption/r/innodb-bad-key-change5.result
@@ -0,0 +1,28 @@
+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+call mtr.add_suppression("Couldn't load plugins from 'file_key_management*");
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	Warning	Table test/t1 is encrypted but encryption service or used key_id is not available.  Can't continue checking table.
+test.t1	optimize	Warning	InnoDB: Cannot defragment table test/t1: returned error code 192
+
+test.t1	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t1	optimize	error	Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
+test.t1	optimize	status	Operation failed
+Warnings:
+Warning	192	Table test/t1 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
+Error	1296	Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
+SHOW WARNINGS;
+Level	Code	Message
+Warning	192	Table test/t1 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
+Error	1296	Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
+DROP TABLE t1;
diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change5.opt b/mysql-test/suite/encryption/t/innodb-bad-key-change5.opt
@@ -0,0 +1,4 @@
+--loose-innodb-buffer-pool-stats
+--loose-innodb-buffer-page
+--loose-innodb-buffer-page-lru
+--innodb-defragment=1
diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change5.test b/mysql-test/suite/encryption/t/innodb-bad-key-change5.test
@@ -0,0 +1,87 @@
+--source include/have_innodb.inc
+# embedded does not support restart
+-- source include/not_embedded.inc
+-- source include/not_valgrind.inc
+# Avoid CrashReporter popup on Mac
+-- source include/not_crashrep.inc
+#
+# MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys
+#
+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+# Suppression for builds where file_key_management plugin is linked statically
+call mtr.add_suppression("Couldn't load plugins from 'file_key_management*");
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys1.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530143
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-defragment=1 --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--let $MYSQLD_TMPDIR = `SELECT @@tmpdir`
+--let $MYSQLD_DATADIR = `SELECT @@datadir`
+
+--disable_query_log
+let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
+let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
+--enable_query_log
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys2.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530144
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-defragment=1 --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+
+OPTIMIZE TABLE t1;
+SHOW WARNINGS;
+
+--remove_file $MYSQLTEST_VARDIR/keys1.txt
+--remove_file $MYSQLTEST_VARDIR/keys2.txt
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys1.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530143
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-defragment=1 --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+DROP TABLE t1;
+--remove_file $MYSQLTEST_VARDIR/keys1.txt
+
+# reset system
+--disable_query_log
+EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
+EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
+--enable_query_log
diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (C) 2013, 2014 Facebook, Inc. All Rights Reserved.
-Copyright (C) 2014, SkySQL Ab. All Rights Reserved.
+Copyright (C) 2014, 2015, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -22,7 +22,7 @@ Index defragmentation.
 
 Created  05/29/2014 Rongrong Zhong
 Modified 16/07/2014 Sunguck Lee
-Modified 30/07/2014 Jan Lindström jan.lindstrom@skysql.com
+Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com
 *******************************************************/
 
 #include "btr0defragment.h"
@@ -208,16 +208,27 @@ synchronized defragmentation. */
 os_event_t
 btr_defragment_add_index(
 	dict_index_t*	index,	/*!< index to be added  */
-	bool		async)	/*!< whether this is an async defragmentation */
+	bool		async,	/*!< whether this is an async
+				defragmentation */
+	dberr_t*	err)	/*!< out: error code */
 {
 	mtr_t mtr;
 	ulint space = dict_index_get_space(index);
 	ulint zip_size = dict_table_zip_size(index->table);
 	ulint page_no = dict_index_get_page(index);
+	*err = DB_SUCCESS;
+
 	mtr_start(&mtr);
 	// Load index rood page.
 	page_t* page = btr_page_get(space, zip_size, page_no,
 				    RW_NO_LATCH, index, &mtr);
+
+	if (page == NULL && index->table->is_encrypted) {
+		mtr_commit(&mtr);
+		*err = DB_DECRYPTION_FAILED;
+		return NULL;
+	}
+
 	if (btr_page_get_level(page, &mtr) == 0) {
 		// Index root is a leaf page, no need to defragment.
 		mtr_commit(&mtr);
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
@@ -12168,21 +12168,29 @@ ha_innobase::defragment_table(
 	const char*	index_name,	/*!< in: index name */
 	bool		async)		/*!< in: whether to wait until finish */
 {
-	char    norm_name[FN_REFLEN];
-	dict_table_t* table;
-	dict_index_t* index;
+	char		norm_name[FN_REFLEN];
+	dict_table_t*	table = NULL;
+	dict_index_t*	index = NULL;
 	ibool		one_index = (index_name != 0);
 	int		ret = 0;
+	dberr_t		err = DB_SUCCESS;
+
 	if (!srv_defragment) {
 		return ER_FEATURE_DISABLED;
 	}
+
 	normalize_table_name(norm_name, name);
+
 	table = dict_table_open_on_name(norm_name, FALSE,
 		FALSE, DICT_ERR_IGNORE_NONE);
+
 	for (index = dict_table_get_first_index(table); index;
 	     index = dict_table_get_next_index(index)) {
-		if (one_index && strcasecmp(index_name, index->name) != 0)
+
+		if (one_index && strcasecmp(index_name, index->name) != 0) {
 			continue;
+		}
+
 		if (btr_defragment_find_index(index)) {
 			// We borrow this error code. When the same index is
 			// already in the defragmentation queue, issue another
@@ -12197,7 +12205,23 @@ ha_innobase::defragment_table(
 			ret = ER_SP_ALREADY_EXISTS;
 			break;
 		}
-		os_event_t event = btr_defragment_add_index(index, async);
+
+		os_event_t event = btr_defragment_add_index(index, async, &err);
+
+		if (err != DB_SUCCESS) {
+			push_warning_printf(
+				current_thd,
+				Sql_condition::WARN_LEVEL_WARN,
+				ER_NO_SUCH_TABLE,
+				"Table %s is encrypted but encryption service or"
+				" used key_id is not available. "
+				" Can't continue checking table.",
+				index->table->name);
+
+			ret = convert_error_code_to_mysql(err, 0, current_thd);
+			break;
+		}
+
 		if (!async && event) {
 			while(os_event_wait_time(event, 1000000)) {
 				if (thd_killed(current_thd)) {
@@ -12208,18 +12232,23 @@ ha_innobase::defragment_table(
 			}
 			os_event_free(event);
 		}
+
 		if (ret) {
 			break;
 		}
+
 		if (one_index) {
 			one_index = FALSE;
 			break;
 		}
 	}
+
 	dict_table_close(table, FALSE, FALSE);
+
 	if (ret == 0 && one_index) {
 		ret = ER_NO_SUCH_INDEX;
 	}
+
 	return ret;
 }
 
@@ -13536,7 +13565,6 @@ ha_innobase::check(
 						"Table %s is encrypted but encryption service or"
 						" used key_id is not available. "
 						" Can't continue checking table.",
-
 						index->table->name);
 				} else {
 					push_warning_printf(
diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h
@@ -259,10 +259,8 @@ btr_block_get_func(
 	ulint		mode,		/*!< in: latch mode */
 	const char*	file,		/*!< in: file name */
 	ulint		line,		/*!< in: line where called */
-# ifdef UNIV_SYNC_DEBUG
-	const dict_index_t*	index,	/*!< in: index tree, may be NULL
+	dict_index_t*	index,		/*!< in: index tree, may be NULL
 					if it is not an insert buffer tree */
-# endif /* UNIV_SYNC_DEBUG */
 	mtr_t*		mtr);		/*!< in/out: mini-transaction */
 # ifdef UNIV_SYNC_DEBUG
 /** Gets a buffer page and declares its latching order level.
@@ -275,7 +273,7 @@ btr_block_get_func(
 @return the block descriptor */
 #  define btr_block_get(space,zip_size,page_no,mode,index,mtr)	\
 	btr_block_get_func(space,zip_size,page_no,mode,		\
-			   __FILE__,__LINE__,index,mtr)
+			   __FILE__,__LINE__,(dict_index_t*)index,mtr)
 # else /* UNIV_SYNC_DEBUG */
 /** Gets a buffer page and declares its latching order level.
 @param space	tablespace identifier
@@ -286,7 +284,8 @@ btr_block_get_func(
 @param mtr	mini-transaction handle
 @return the block descriptor */
 #  define btr_block_get(space,zip_size,page_no,mode,idx,mtr)		\
-	btr_block_get_func(space,zip_size,page_no,mode,__FILE__,__LINE__,mtr)
+		btr_block_get_func(space,zip_size,page_no,mode, \
+			__FILE__,__LINE__,(dict_index_t*)idx,mtr)
 # endif /* UNIV_SYNC_DEBUG */
 /** Gets a buffer page and declares its latching order level.
 @param space	tablespace identifier
@@ -297,7 +296,8 @@ btr_block_get_func(
 @param mtr	mini-transaction handle
 @return the uncompressed page frame */
 # define btr_page_get(space,zip_size,page_no,mode,idx,mtr)		\
-	buf_block_get_frame(btr_block_get(space,zip_size,page_no,mode,idx,mtr))
+	buf_block_get_frame(btr_block_get(space,zip_size,page_no, \
+			mode,(dict_index_t*)idx,mtr))
 #endif /* !UNIV_HOTBACKUP */
 /**************************************************************//**
 Gets the index id field of a page.
diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic
@@ -49,16 +49,19 @@ btr_block_get_func(
 	ulint		mode,		/*!< in: latch mode */
 	const char*	file,		/*!< in: file name */
 	ulint		line,		/*!< in: line where called */
-#ifdef UNIV_SYNC_DEBUG
-	const dict_index_t*	index,	/*!< in: index tree, may be NULL
+	dict_index_t*	index,		/*!< in: index tree, may be NULL
 					if it is not an insert buffer tree */
-#endif /* UNIV_SYNC_DEBUG */
 	mtr_t*		mtr)		/*!< in/out: mtr */
 {
 	buf_block_t*	block;
+	dberr_t		err;
 
 	block = buf_page_get_gen(space, zip_size, page_no, mode,
-		NULL, BUF_GET, file, line, mtr);
+		NULL, BUF_GET, file, line, mtr, &err);
+
+	if (err == DB_DECRYPTION_FAILED) {
+		index->table->is_encrypted = true;
+	}
 
 	if (block) {
 		if (mode != RW_NO_LATCH) {
diff --git a/storage/innobase/include/btr0defragment.h b/storage/innobase/include/btr0defragment.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (C) 2013, 2014 Facebook, Inc. All Rights Reserved.
+Copyright (C) 2014, 2015, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -67,7 +68,9 @@ is a synchronized defragmentation. */
 os_event_t
 btr_defragment_add_index(
 	dict_index_t*	index,	/*!< index to be added  */
-	bool		async);	/*!< whether this is an async defragmentation */
+	bool		async,	/*!< whether this is an async
+				defragmentation */
+	dberr_t*	err);	/*!< out: error code */
 /******************************************************************//**
 When table is dropped, this function is called to mark a table as removed in
 btr_efragment_wq. The difference between this function and the remove_index
diff --git a/storage/xtradb/btr/btr0defragment.cc b/storage/xtradb/btr/btr0defragment.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (C) 2012, 2014 Facebook, Inc. All Rights Reserved.
-Copyright (C) 2014, SkySQL Ab. All Rights Reserved.
+Copyright (C) 2014, 2015, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -22,7 +22,7 @@ Index defragmentation.
 
 Created  05/29/2014 Rongrong Zhong
 Modified 16/07/2014 Sunguck Lee
-Modified 30/07/2014 Jan Lindström jan.lindstrom@skysql.com
+Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com
 *******************************************************/
 
 #include "btr0defragment.h"
@@ -208,16 +208,27 @@ synchronized defragmentation. */
 os_event_t
 btr_defragment_add_index(
 	dict_index_t*	index,	/*!< index to be added  */
-	bool		async)	/*!< whether this is an async defragmentation */
+	bool		async,	/*!< whether this is an async
+				defragmentation */
+	dberr_t*	err)	/*!< out: error code */
 {
 	mtr_t mtr;
 	ulint space = dict_index_get_space(index);
 	ulint zip_size = dict_table_zip_size(index->table);
 	ulint page_no = dict_index_get_page(index);
+	*err = DB_SUCCESS;
+
 	mtr_start(&mtr);
 	// Load index rood page.
 	page_t* page = btr_page_get(space, zip_size, page_no,
 				    RW_NO_LATCH, index, &mtr);
+
+	if (page == NULL && index->table->is_encrypted) {
+		mtr_commit(&mtr);
+		*err = DB_DECRYPTION_FAILED;
+		return NULL;
+	}
+
 	if (btr_page_get_level(page, &mtr) == 0) {
 		// Index root is a leaf page, no need to defragment.
 		mtr_commit(&mtr);
diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc
diff --git a/storage/xtradb/include/btr0btr.h b/storage/xtradb/include/btr0btr.h
diff --git a/storage/xtradb/include/btr0btr.ic b/storage/xtradb/include/btr0btr.ic
diff --git a/storage/xtradb/include/btr0defragment.h b/storage/xtradb/include/btr0defragment.h
