Bug#28663404 GENERAL TABLESPACE ENCRYPTION USES SAME TABLESOACE KEY

             IN SUBSEQUENT ENCRYPTIONS

Issue:
 Subsequent encryption operation on a general tablespace
 were using same tablespace encryption key.
 Read bug page for complete details.

Fix:
 This fix ensures that every subsequent encryption uses new
 tablespace encryption key.

RB: 20566
Reviewed by : Allen Lai <zheng.lai@oracle.com>

mysql-test/suite/innodb/r/tablespace_encrypt_7.result

@@ -25,6 +25,7 @@ SOME VALUES
 SOME VALUES
 ALTER TABLESPACE encrypt_ts ENCRYPTION='Y';
 ERROR HY000: Can't find master key from keyring, please check in the server log if a keyring plugin is loaded and initialized successfully.
+#-------------------------- TEST 1 -------------------------------------#
 #########################################################################
 # RESTART 1 : WITH KEYRING PLUGIN
 #########################################################################
@@ -221,6 +222,7 @@ SOME VALUES
 SOME VALUES
 SOME VALUES
 SOME VALUES
+#-------------------------- TEST 2 -------------------------------------#
 #########################################################################
 # RESTART 9 : WITH KEYRING PLUGIN
 #########################################################################
@@ -241,7 +243,7 @@ SOME VALUES
 SOME VALUES
 ########################################################################
 # ALTER TABLESPACE 5 :  Encrypted => unencrypted                       #
-#                       crash just before resetting progress onpage 0  #
+#                       crash just before updating ts flags on page0   #
 ########################################################################
 ALTER TABLESPACE encrypt_ts ENCRYPTION='Y';
 # Set process to crash just after flushing page 0 at the end
@@ -349,8 +351,43 @@ SOME VALUES
 SOME VALUES
 SOME VALUES
 SOME VALUES
+#-------------------------- TEST 3 -------------------------------------#
+#########################################################################
+# RESTART 13 : WITH KEYRING PLUGIN
+#########################################################################
+DROP TABLE t1;
+DROP TABLESPACE encrypt_ts;
+CREATE TABLESPACE encrypt_ts ADD DATAFILE 'encrypt_ts.ibd';
+CREATE TABLE t1 (C CHAR(10)) TABLESPACE=encrypt_ts;
+SET GLOBAL innodb_limit_optimistic_insert_debug=2;
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+# Make sure checkpoint is not moved
+SET GLOBAL innodb_log_checkpoint_now = ON;
+SET GLOBAL innodb_page_cleaner_disabled_debug = 1;
+SET GLOBAL innodb_dict_stats_disabled_debug = 1;
+SET GLOBAL innodb_master_thread_disabled_debug = 1;
+# Following encryption will create a new tablespace key (KEY1)
+# KEY1 will be written on REDO log
+ALTER TABLESPACE encrypt_ts encryption='Y';
+# Following unencryption will remove tablespace key
+ALTER TABLESPACE encrypt_ts encryption='N';
+SET SESSION debug= '+d,alter_encrypt_tablespace_page_6';
+SET SESSION debug= '+d,flush_each_dirtied_page';
+# Following encryption will create a new tablespace key (KEY2)
+# KEY2 will be written on REDO log
+# Flush dirtied pages encrypted with KEY2 before crash
+ALTER TABLESPACE encrypt_ts encryption='Y';
+# Restart after crash
 ###########
 # Cleanup #
 ###########
 DROP TABLE t1;
 DROP TABLESPACE encrypt_ts;
+SET SESSION debug= '-d,alter_encrypt_tablespace_page_6';
+SET SESSION debug= '-d,flush_each_dirtied_page';

mysql-test/suite/innodb/t/tablespace_encrypt_7.test

@@ -1,12 +1,16 @@
 ################################################################################
 # InnoDB transparent tablespace data encryption for general shared tablespace.
 # This test case will test
-#    - Crash
-#        - just before flushing page 0 at the end of (un)encryption
-#        - just after flushing page 0 at the end of (un)encryption
-#    -Crash
-#        - just before updating tablespace flags on page 0
-#        - just before resetting progress on page 0
+#    - Test 1
+#        - 1.1 : Crash just before flushing page 0 at the end of (un)encryption
+#        - 1.2 : Crash just after flushing page 0 at the end of (un)encryption
+#    - Test 2
+#        - 2.1 : Crash just before updating tablespace flags on page 0
+#        - 2.2 : Crash just before resetting progress on page 0
+#    - Test 3
+#        - A scneario in which tablespace key found during REDO log scan is
+#          old whereas tablespace key on disk is new. Here, REDO log scan key
+#          should be discarded and new tablespace key should be used instead.
 ################################################################################
 
 --source include/big_test.inc
@@ -54,6 +58,8 @@ SELECT * FROM t1 LIMIT 10;
 --error ER_CANNOT_FIND_KEY_IN_KEYRING
 ALTER TABLESPACE encrypt_ts ENCRYPTION='Y';
 
+--echo #-------------------------- TEST 1 -------------------------------------#
+
 --echo #########################################################################
 --echo # RESTART 1 : WITH KEYRING PLUGIN
 --echo #########################################################################
@@ -236,6 +242,8 @@ SELECT NAME, ENCRYPTION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES WHERE NAME='e
 # be unencrypted now.
 SELECT * FROM t1 LIMIT 10;
 
+--echo #-------------------------- TEST 2 -------------------------------------#
+
 --echo #########################################################################
 --echo # RESTART 9 : WITH KEYRING PLUGIN
 --echo #########################################################################
@@ -247,7 +255,7 @@ SELECT * FROM t1 LIMIT 10;
 
 --echo ########################################################################
 --echo # ALTER TABLESPACE 5 :  Encrypted => unencrypted                       #
---echo #                       crash just before resetting progress onpage 0  #
+--echo #                       crash just before updating ts flags on page0   #
 --echo ########################################################################
 ALTER TABLESPACE encrypt_ts ENCRYPTION='Y';
 --echo # Set process to crash just after flushing page 0 at the end
@@ -332,10 +340,61 @@ SELECT NAME, ENCRYPTION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES WHERE NAME='e
 # be unencrypted now.
 SELECT * FROM t1 LIMIT 10;
 
+--echo #-------------------------- TEST 3 -------------------------------------#
+
+--echo #########################################################################
+--echo # RESTART 13 : WITH KEYRING PLUGIN
+--echo #########################################################################
+let $restart_parameters = restart: --early-plugin-load=keyring_file=$KEYRING_PLUGIN --loose-keyring_file_data=$MYSQL_TMP_DIR/mysecret_keyring $KEYRING_PLUGIN_OPT --innodb_doublewrite=0;
+--source include/restart_mysqld_no_echo.inc
+
+DROP TABLE t1;
+DROP TABLESPACE encrypt_ts;
+CREATE TABLESPACE encrypt_ts ADD DATAFILE 'encrypt_ts.ibd';
+CREATE TABLE t1 (C CHAR(10)) TABLESPACE=encrypt_ts;
+SET GLOBAL innodb_limit_optimistic_insert_debug=2;
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+INSERT INTO t1 VALUES("SOMEVALUE");
+
+--echo # Make sure checkpoint is not moved
+SET GLOBAL innodb_log_checkpoint_now = ON;
+SET GLOBAL innodb_page_cleaner_disabled_debug = 1;
+SET GLOBAL innodb_dict_stats_disabled_debug = 1;
+SET GLOBAL innodb_master_thread_disabled_debug = 1;
+
+--echo # Following encryption will create a new tablespace key (KEY1)
+--echo # KEY1 will be written on REDO log
+ALTER TABLESPACE encrypt_ts encryption='Y';
+
+--echo # Following unencryption will remove tablespace key
+ALTER TABLESPACE encrypt_ts encryption='N';
+
+# Force encryption to crash after page 6 is encrypted
+SET SESSION debug= '+d,alter_encrypt_tablespace_page_6';
+# Force every page to be flushed after its dirtied
+SET SESSION debug= '+d,flush_each_dirtied_page';
+
+--echo # Following encryption will create a new tablespace key (KEY2)
+--echo # KEY2 will be written on REDO log
+--echo # Flush dirtied pages encrypted with KEY2 before crash
+--source include/expect_crash.inc
+--error 0,CR_SERVER_LOST,ER_INTERNAL_ERROR
+ALTER TABLESPACE encrypt_ts encryption='Y';
+--echo # Restart after crash
+# Without fix, following restart would crash reporting page corruption
+let $restart_parameters = restart: --early-plugin-load=keyring_file=$KEYRING_PLUGIN --loose-keyring_file_data=$MYSQL_TMP_DIR/mysecret_keyring $KEYRING_PLUGIN_OPT --innodb_doublewrite=0 --debug=d,dont_update_key_found_during_REDO_scan;
+--source include/start_mysqld_no_echo.inc
+
 --echo ###########
 --echo # Cleanup #
 --echo ###########
-
 DROP TABLE t1;
 DROP TABLESPACE encrypt_ts;
+SET SESSION debug= '-d,alter_encrypt_tablespace_page_6';
+SET SESSION debug= '-d,flush_each_dirtied_page';
 remove_file $MYSQL_TMP_DIR/mysecret_keyring;

storage/innobase/fil/fil0fil.cc

@@ -8794,9 +8794,43 @@ static void fil_tablespace_encryption_init(const fil_space_t *space) {
       continue;
     }
 
-    dberr_t err;
+    dberr_t err = DB_SUCCESS;
 
-    err = fil_set_encryption(space->id, Encryption::AES, key.ptr, key.iv);
+    ut_ad(!fsp_is_system_tablespace(space->id));
+
+    /* Here we try to populate space tablespace_key which is read during
+    REDO scan.
+
+    Consider following scenario:
+    1. Alter tablespce .. encrypt=y (KEY1)
+    2. Alter tablespce .. encrypt=n
+    3. Alter tablespce .. encrypt=y (KEY2)
+
+    Lets say there is a crash after (3) is finished successfully. All the pages
+    of tablespace are encrypted with KEY2.
+
+    During recovery:
+    ----------------
+    - Let's say we scanned till REDO of (1) but couldn't reach to REDO of (3).
+    - So we've got tablespace key as KEY1.
+    - Note, tablespace pages were encrypted using KEY2 which would have been
+      found on page 0 and thus loaded already in file_space_t.
+
+    If we overwrite this space key (KEY2) with the one we got from REDO log
+    scan (KEY1), then when we try to read a page from Disk, we will try to
+    decrypt it using KEY1 whereas page was encrypted with KEY2. ERROR.
+
+    Therefore, for a general tablespace, if tablespace key is already populated
+    it is the latest key and should be used instead of the one read during
+    REDO log scan.
+
+    For file-per-table tablespace, which is not INPLACE algorithm, copy what
+    is found on REDO Log.
+    */
+    if (fsp_is_file_per_table(space->id, space->flags) ||
+        (space->encryption_key == nullptr && space->encryption_iv == nullptr)) {
+      err = fil_set_encryption(space->id, Encryption::AES, key.ptr, key.iv);
+    }
 
     if (err != DB_SUCCESS) {
       ib::error(ER_IB_MSG_343) << "Can't set encryption information"
@@ -9546,6 +9580,11 @@ byte *fil_tablespace_redo_encryption(byte *ptr, const byte *end,
       if (recv_key.space_id == space_id) {
         iv = recv_key.iv;
         key = recv_key.ptr;
+
+        DBUG_EXECUTE_IF(
+            "dont_update_key_found_during_REDO_scan",
+            key = static_cast<byte *>(ut_malloc_nokey(ENCRYPTION_KEY_LEN));
+            iv = static_cast<byte *>(ut_malloc_nokey(ENCRYPTION_KEY_LEN)););
       }
     }
 

storage/innobase/fsp/fsp0fsp.cc

@@ -869,53 +869,6 @@ encryption_op_type fsp_header_encryption_op_type_in_progress(
   return (op);
 }
 
-/** Get encryption information from page 0 of tablespace.
-@param[in]	space_id	space id.
-@param[in]	space_flags	space flags
-@param[out]	key		tablespace encryption key
-@param[out]	iv		tablespace encryption iv
-@return true, if information read successfully
-*/
-static bool fsp_header_read_encryption_info(space_id_t space_id,
-                                            ulint space_flags, byte *key,
-                                            byte *iv) {
-  buf_block_t *block;
-  ulint offset;
-  page_t *page;
-  mtr_t mtr;
-
-  const page_size_t page_size(space_flags);
-
-  mtr_start(&mtr);
-  /* Read encryption info from page 0. */
-  block = buf_page_get(page_id_t(space_id, 0), page_size, RW_SX_LATCH, &mtr);
-  if (block == nullptr) {
-    return (false);
-  }
-
-  buf_block_dbg_add_level(block, SYNC_FSP_PAGE);
-  ut_ad(space_id == page_get_space_id(buf_block_get_frame(block)));
-
-  offset = fsp_header_get_encryption_offset(page_size);
-  ut_ad(offset != 0 && offset < UNIV_PAGE_SIZE);
-
-  page = buf_block_get_frame(block);
-
-  if (!Encryption::decode_encryption_info(key, iv, page + offset, false)) {
-    mtr_commit(&mtr);
-    return (false);
-  }
-
-  mtr_commit(&mtr);
-  byte buf[ENCRYPTION_KEY_LEN];
-  memset(buf, 0, ENCRYPTION_KEY_LEN);
-  if (memcmp(key, buf, ENCRYPTION_KEY_LEN) == 0) {
-    return (false);
-  }
-
-  return (true);
-}
-
 /** Write the encryption info into the space header.
 @param[in]      space_id		tablespace id
 @param[in]      space_flags		tablespace flags
@@ -969,16 +922,8 @@ bool fsp_header_write_encryption(space_id_t space_id, ulint space_flags,
     }
   }
 
-  /* For user tablespace, don't erase encryption informtaion from page 0 */
-  if (fsp_is_ibd_tablespace(space_id)) {
-    byte buf[ENCRYPTION_INFO_SIZE];
-    memset(buf, 0, ENCRYPTION_INFO_SIZE);
-    if (memcmp(encrypt_info, buf, ENCRYPTION_INFO_SIZE) != 0) {
-      mlog_write_string(page + offset, encrypt_info, ENCRYPTION_INFO_SIZE, mtr);
-    }
-  } else {
-    mlog_write_string(page + offset, encrypt_info, ENCRYPTION_INFO_SIZE, mtr);
-  }
+  /* Write encryption info passed */
+  mlog_write_string(page + offset, encrypt_info, ENCRYPTION_INFO_SIZE, mtr);
 
   return (true);
 }
@@ -4063,6 +4008,10 @@ static void mark_all_page_dirty_in_tablespace(THD *thd, space_id_t space_id,
       DEBUG_SYNC(thd, "alter_encrypt_tablespace_wait_after_page5");
     }
 #endif /* UNIV_DEBUG */
+
+    DBUG_EXECUTE_IF("flush_each_dirtied_page",
+                    buf_LRU_flush_or_remove_pages(
+                        space_id, BUF_REMOVE_FLUSH_WRITE, 0, false););
   }
 }
 
@@ -4101,22 +4050,12 @@ dberr_t fsp_alter_encrypt_tablespace(THD *thd, space_id_t space_id,
       /* Assert that tablespace is not encrypted */
       ut_ad(!FSP_FLAGS_GET_ENCRYPTION(space->flags));
 
-      /* Fill key, iv and prepare encryption_info to be
-      written in page 0 */
+      /* Fill key, iv and prepare encryption_info to be written in page 0 */
       byte key[ENCRYPTION_KEY_LEN];
       byte iv[ENCRYPTION_KEY_LEN];
 
-      /* Try to read encryption information from page 0. If found, that will be
-      used, otherwise a new encryption key, iv will be generated and used. */
-      if (fsp_header_read_encryption_info(space->id, space->flags,
-                                          space->encryption_key,
-                                          space->encryption_iv)) {
-        memcpy(key, space->encryption_key, ENCRYPTION_KEY_LEN);
-        memcpy(iv, space->encryption_iv, ENCRYPTION_KEY_LEN);
-      } else {
-        Encryption::random_value(key);
-        Encryption::random_value(iv);
-      }
+      Encryption::random_value(key);
+      Encryption::random_value(iv);
 
       /* Prepare encrypted encryption information to be written on page 0. */
       if (!Encryption::fill_encryption_info(key, iv, encryption_info, false)) {

storage/innobase/include/os0file.h

@@ -404,10 +404,9 @@ struct Encryption {
   @param[in,out]	key		key
   @param[in,out]	iv		iv
   @param[in]		encryption_info	encryption info
-  @param[in]		report		true, if error to be reported
   @return true if success */
-  static bool decode_encryption_info(byte *key, byte *iv, byte *encryption_info,
-                                     const bool report = true);
+  static bool decode_encryption_info(byte *key, byte *iv,
+                                     byte *encryption_info);
 
   /** Encrypt the redo log block.
   @param[in]	type		IORequest

storage/innobase/os/os0file.cc

@@ -8226,11 +8226,9 @@ byte *Encryption::get_master_key_from_info(byte *encrypt_info, Version version,
 @param[in,out]	key		key
 @param[in,out]	iv		iv
 @param[in]	encryption_info	encryption info
-@param[in]	report		true, if error to be rerpoted
 @return true if success */
 bool Encryption::decode_encryption_info(byte *key, byte *iv,
-                                        byte *encryption_info,
-                                        const bool report) {
+                                        byte *encryption_info) {
   byte *ptr;
   byte *master_key = nullptr;
   uint32 m_key_id;
@@ -8262,11 +8260,8 @@ bool Encryption::decode_encryption_info(byte *key, byte *iv,
       return (true);
     }
 
-    if (report) {
-      ib::error(ER_IB_MSG_837) << "Failed to decrypt encryption information,"
-                               << " found unexpected version of it!";
-    }
-
+    ib::error(ER_IB_MSG_837) << "Failed to decrypt encryption information,"
+                             << " found unexpected version of it!";
     return (false);
   }