WL#12079: Encrypt binary log caches at rest

Use AES with CTR mode to encrypt temporary files created when
binary log caches spill to disk when binlog_encryption is on.
Client sessions will have their binary log caches unencrypted
when binlog_encryption is off and client sessions will have
their binary log caches encrypted when binlog_encryption is on.
Every time a transaction commits, it requests its binlog cache
to be reset. On this reset operation, the binlog cache shall
check `binlog_encryption` value to enable or disable its
encryption, and close the ciphers and open them again using
new password and IV when binlog_encryption is on.

RB: 21577

Author: Daogang Qu

client/CMakeLists.txt

@@ -86,7 +86,7 @@ SET(MYSQLBINLOG_LIB_SOURCES
   ${CMAKE_SOURCE_DIR}/sql/basic_istream.cc
   ${CMAKE_SOURCE_DIR}/sql/binlog_istream.cc
   ${CMAKE_SOURCE_DIR}/sql/binlog_reader.cc
-  ${CMAKE_SOURCE_DIR}/sql/rpl_cipher.cc
+  ${CMAKE_SOURCE_DIR}/sql/stream_cipher.cc
   ${CMAKE_SOURCE_DIR}/sql/rpl_log_encryption.cc
 )
 ADD_LIBRARY(mysqlbinlog_lib STATIC ${MYSQLBINLOG_LIB_SOURCES})

include/my_sys.h

@@ -64,6 +64,7 @@
 #include "mysql/components/services/psi_memory_bits.h"
 #include "mysql/components/services/psi_stage_bits.h"
 #include "mysql/psi/psi_base.h"
+#include "sql/stream_cipher.h"
 
 struct CHARSET_INFO;
 struct MY_CHARSET_LOADER;
@@ -472,6 +473,10 @@ struct IO_CACHE /* Used when cacheing files */
     somewhere else
   */
   bool alloced_buffer{false};
+  // This is an encryptor for encrypting the temporary file of the IO cache.
+  Stream_cipher *m_encryptor = nullptr;
+  // This is a decryptor for decrypting the temporary file of the IO cache.
+  Stream_cipher *m_decryptor = nullptr;
 };
 
 typedef int (*qsort2_cmp)(const void *, const void *, const void *);
@@ -975,4 +980,69 @@ extern MYSQL_FILE *mysql_stdin;
   @} (end of group MYSYS)
 */
 
+// True if the temporary file of binlog cache is encrypted.
+#ifndef DBUG_OFF
+extern bool binlog_cache_temporary_file_is_encrypted;
+#endif
+
+/**
+  This is a wrapper around mysql_file_seek. Seek to a position in the
+  temporary file of a binlog cache, and set the encryption/decryption
+  stream offset if binlog_encryption is on.
+
+  @param cache The handler of a binlog cache to seek.
+  @param pos The expected position (absolute or relative)
+  @param whence A direction parameter and one of
+                {SEEK_SET, SEEK_CUR, SEEK_END}
+  @param flags  The bitmap of different flags
+                MY_WME | MY_FAE | MY_NABP | MY_FNABP |
+                MY_DONT_CHECK_FILESIZE and so on.
+
+  @retval The new position in the file, or MY_FILEPOS_ERROR on error.
+*/
+my_off_t mysql_encryption_file_seek(IO_CACHE *cache, my_off_t pos, int whence,
+                                    myf flags);
+/**
+   This is a wrapper around mysql_file_read. Read data from the temporary
+   file of a binlog cache, and take care of decrypting the data if
+   binlog_encryption is on.
+
+
+   @param cache The handler of a binlog cache to read.
+   @param[out] buffer The memory buffer to write to.
+   @param count The length of data in the temporary file to be read in bytes.
+   @param flags The bitmap of different flags
+                MY_WME | MY_FAE | MY_NABP | MY_FNABP |
+                MY_DONT_CHECK_FILESIZE and so on.
+
+   @retval The length of bytes to be read, or MY_FILE_ERROR on error.
+*/
+size_t mysql_encryption_file_read(IO_CACHE *cache, uchar *buffer, size_t count,
+                                  myf flags);
+/**
+   This is a wrapper around mysql_file_write. Write data in buffer to the
+   temporary file of a binlog cache, and take care of encrypting the data
+   if binlog_encryption is on.
+
+   @param cache The handler of a binlog cache to write.
+   @param buffer The memory buffer to write from.
+   @param count The length of data in buffer to be written in bytes.
+   @param flags The bitmap of different flags
+                MY_WME | MY_FAE | MY_NABP | MY_FNABP |
+                MY_DONT_CHECK_FILESIZE and so on
+
+   if (flags & (MY_NABP | MY_FNABP)) {
+     @retval 0 if count == 0
+     @retval 0 success
+     @retval MY_FILE_ERROR error
+   } else {
+     @retval 0 if count == 0
+     @retval The number of bytes written on success.
+     @retval MY_FILE_ERROR error
+     @retval The actual number of bytes written on partial success (if
+             less than count bytes were written).
+   }
+*/
+size_t mysql_encryption_file_write(IO_CACHE *cache, const uchar *buffer,
+                                   size_t count, myf flags);
 #endif /* _my_sys_h */

mysql-test/suite/binlog/r/binlog_cache_temp_file_encrypt_cover.result

@@ -0,0 +1,9 @@
+CREATE TABLE t1 (c1 TEXT);
+# Adding debug point 'verify_mysql_encryption_file_write_bytes' to @@GLOBAL.debug
+INSERT INTO t1 VALUES (REPEAT('123', 16384.0));
+# Removing debug point 'verify_mysql_encryption_file_write_bytes' from @@GLOBAL.debug
+# Adding debug point 'simulate_binlog_cache_temp_file_encrypt_fail' to @@GLOBAL.debug
+INSERT INTO t1 VALUES (REPEAT('456', 16384.0));
+ERROR HY000: Error writing file 'binlog' ((errno: #)
+# Removing debug point 'simulate_binlog_cache_temp_file_encrypt_fail' from @@GLOBAL.debug
+DROP TABLE t1;

mysql-test/suite/binlog/t/binlog_cache_temp_file_encrypt_cover-master.opt

@@ -0,0 +1,4 @@
+--early-plugin-load="keyring_file=$KEYRING_PLUGIN"
+--keyring_file_data=$MYSQL_TMP_DIR/keyring_data
+--binlog_encryption=ON
+$KEYRING_PLUGIN_OPT

mysql-test/suite/binlog/t/binlog_cache_temp_file_encrypt_cover.test

@@ -0,0 +1,49 @@
+# ==== Purpose ====
+#
+# Verify that the correct number of bytes are written by
+# mysql_encryption_file_write(...) if both MY_NABP and
+# MY_FNABP are not set.
+#
+# ==== Implementation ====
+#
+# 1. Start the master with binlog_encryption on.
+# 2. Create a table t1 with a TEXT column.
+# 3. Execute a trx to insert a big text into the table to make
+#    binlog cache spill to disk and ensure that the correct
+#    number of bytes are written by mysql_encryption_file_write(...).
+# 4. Verify that an error 'ER_ERROR_ON_WRITE' is reported if there
+#    is a failure on encrypting binlog cache temporary file.
+# ==== References ====
+#
+# Wl#12079  Encrypt binary log caches at rest
+
+# This test script will be run only in non GR set up.
+--source include/not_group_replication_plugin.inc
+--source include/have_binlog_format_row.inc
+# Restrict the test runs to only debug builds, since we set DEBUG point in the test.
+--source include/have_debug.inc
+
+--let $data_size= `select 0.5 * @@global.binlog_cache_size`
+
+CREATE TABLE t1 (c1 TEXT);
+
+--let $debug_point= verify_mysql_encryption_file_write_bytes
+--source include/add_debug_point.inc
+
+eval INSERT INTO t1 VALUES (REPEAT('123', $data_size));
+
+--let $debug_point= verify_mysql_encryption_file_write_bytes
+--source include/remove_debug_point.inc
+
+
+--let $debug_point= simulate_binlog_cache_temp_file_encrypt_fail
+--source include/add_debug_point.inc
+
+--replace_regex /(errno: .*)/(errno: #)/
+--error ER_ERROR_ON_WRITE
+eval INSERT INTO t1 VALUES (REPEAT('456', $data_size));
+
+--let $debug_point= simulate_binlog_cache_temp_file_encrypt_fail
+--source include/remove_debug_point.inc
+
+DROP TABLE t1;

mysql-test/suite/group_replication/r/gr_key_rotation.result

@@ -73,10 +73,21 @@ include/diff_tables.inc [server1:t1, server2:t1]
 # check that t2 exists and has same values in both servers
 include/diff_tables.inc [server1:t2, server2:t2]
 # check that t3 exists and has same values in both servers
+[connection server1]
+SET GLOBAL binlog_encryption=ON;
+CREATE TABLE t4 (c1 TEXT,c2 INT AUTO_INCREMENT PRIMARY KEY);
+# Adding debug point 'ensure_binlog_cache_temporary_file_is_encrypted' to @@GLOBAL.debug
+INSERT INTO t4(c1) VALUES (REPEAT('123', 16384.0));
+# Removing debug point 'ensure_binlog_cache_temporary_file_is_encrypted' from @@GLOBAL.debug
+SET GLOBAL binlog_encryption = OFF;
+include/rpl_sync.inc
+# check that t4 exists and has same values in both servers
+[connection server2]
 UNINSTALL PLUGIN keyring_file;
 [connection server1]
 UNINSTALL PLUGIN keyring_file;
 DROP TABLE t2;
 DROP TABLE t3;
 DROP TABLE t1;
+DROP TABLE t4;
 include/group_replication_end.inc

mysql-test/suite/group_replication/t/gr_key_rotation.test

@@ -14,10 +14,20 @@
 #	2.3 Check that M2 is in ERROR state.
 #	2.4 Stop GR, Install keyring plugin and start GR on M2.
 #	2.5 Check that t3 exists and has same values on both the servers.
-# 3. Clean Up.
+# 3. Verify that the temporary file of binlog cache is encrypted when
+#    the binlog cache spills to disk if binlog_encryption is on.
+#       3.1 Enable binlog_encryption on M1.
+#       3.2 Create a table t4 with TEXT column.
+#       3.3 Execute a trx to insert a big text into the table to make
+#           binlog cache spill to disk and ensure that the temporary
+#           file of binlog cache is encrypted.
+#       3.4 Check that t4 exists and has same values on both the servers.
+# 4. Clean Up.
 ###############################################################################
 
 --source include/big_test.inc
+# Restrict the test runs to only debug builds, since we set DEBUG point in the test.
+--source include/have_debug.inc
 # Ensure that plugin is installed.
 --source include/have_group_replication_plugin_base.inc
 
@@ -176,9 +186,38 @@ DELETE FROM t1 WHERE c1=1;
 --echo # check that t3 exists and has same values in both servers
 --let $diff_tables=server1:t3, server2:t3
 
-# Clean Up
+# Scenario 3
+
+# 3.1 Enable binlog_encryption on M1.
+--let $rpl_connection_name= server1
+--source include/rpl_connection.inc
+
+SET GLOBAL binlog_encryption=ON;
+
+# 3.2 Create a table t4 with TEXT column.
+--let $data_size= `select 0.5 * @@global.binlog_cache_size`
+
+CREATE TABLE t4 (c1 TEXT,c2 INT AUTO_INCREMENT PRIMARY KEY);
+
+# 3.3 Execute a trx to insert a big text into the table to make binlog cache spill to disk
+--let $debug_point= ensure_binlog_cache_temporary_file_is_encrypted
+--source include/add_debug_point.inc
+
+eval INSERT INTO t4(c1) VALUES (REPEAT('123', $data_size));
+
+--let $debug_point= ensure_binlog_cache_temporary_file_is_encrypted
+--source include/remove_debug_point.inc
+
+SET GLOBAL binlog_encryption = OFF;
+
+# 3.4 Check that t4 exists and has same values on both the servers.
+--source include/rpl_sync.inc
+--echo # check that t4 exists and has same values in both servers
+--let $diff_tables=server1:t4, server2:t4
 
 # Uninstall keyring plugin and remove dummy keyring file on server2
+--let $rpl_connection_name= server2
+--source include/rpl_connection.inc
 UNINSTALL PLUGIN keyring_file;
 --remove_file $MYSQL_TMP_DIR/mydummy_key2
 
@@ -191,6 +230,7 @@ UNINSTALL PLUGIN keyring_file;
 DROP TABLE t2;
 DROP TABLE t3;
 DROP TABLE t1;
+DROP TABLE t4;
 
 --source include/force_restart.inc
 --source include/group_replication_end.inc

mysql-test/suite/rpl/r/rpl_binlog_cache_encryption.result

@@ -0,0 +1,31 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+[connection slave]
+[connection master]
+# Part 1 - binlog_encryption = OFF
+CREATE TABLE t1 (c1 INT PRIMARY KEY AUTO_INCREMENT, c2 TEXT);
+# Inserting 50 transactions with save points
+SHOW BINLOG EVENTS shall not fail on the un-encrypted binlog file
+SHOW BINLOG EVENTS IN 'PLAIN_MASTER_FILE';
+# Restarting the server with "--binlog_encryption=ON" and keyring
+include/rpl_restart_server.inc [server_number=1]
+include/assert.inc [binlog_encryption option shall be ON]
+include/assert.inc [Binary log is encrypted using 1st master key]
+# Part 2 - binlog_encryption = ON
+# Inserting 50 transactions with save points
+SHOW BINLOG EVENTS shall not fail on the encrypted binlog file
+SHOW BINLOG EVENTS IN 'ENCRYPTED_MASTER_FILE';
+# Restarting the server without "--binlog_encryption=ON"
+include/rpl_restart_server.inc [server_number=1]
+[connection slave]
+include/start_slave.inc
+[connection master]
+include/sync_slave_sql_with_master.inc
+include/diff_tables.inc [master:t1, slave:t1]
+[connection master]
+UNINSTALL PLUGIN keyring_file;
+DROP TABLE t1;
+include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_binlog_cache_temp_file_encryption.result

@@ -0,0 +1,34 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+CREATE TABLE t1 (c1 TEXT) ENGINE=INNODB;
+CREATE TABLE t2 (c2 TEXT) ENGINE=MYISAM;
+# Adding debug point 'ensure_binlog_cache_temporary_file_is_encrypted' to @@GLOBAL.debug
+INSERT INTO t1 VALUES (REPEAT('123', 16384.0));
+INSERT INTO t2 VALUES (REPEAT('123', 16384.0));
+# Removing debug point 'ensure_binlog_cache_temporary_file_is_encrypted' from @@GLOBAL.debug
+SET GLOBAL binlog_encryption=OFF;
+# Adding debug point 'ensure_binlog_cache_temp_file_encryption_is_disabled' to @@GLOBAL.debug
+INSERT INTO t1 VALUES (REPEAT('off', 16384.0));
+INSERT INTO t2 VALUES (REPEAT('off', 16384.0));
+# Removing debug point 'ensure_binlog_cache_temp_file_encryption_is_disabled' from @@GLOBAL.debug
+SET GLOBAL binlog_encryption=ON;
+# Adding debug point 'ensure_binlog_cache_temporary_file_is_encrypted' to @@GLOBAL.debug
+INSERT INTO t1 VALUES (REPEAT('on1', 16384.0));
+INSERT INTO t2 VALUES (REPEAT('on1', 16384.0));
+# Removing debug point 'ensure_binlog_cache_temporary_file_is_encrypted' from @@GLOBAL.debug
+# Adding debug point 'ensure_binlog_cache_is_reset' to @@GLOBAL.debug
+INSERT INTO t1 VALUES ("567");
+BEGIN;
+INSERT INTO t1 VALUES ("789");
+ROLLBACK;
+INSERT INTO t2 VALUES ("567");
+# Removing debug point 'ensure_binlog_cache_is_reset' from @@GLOBAL.debug
+include/sync_slave_sql_with_master.inc
+include/diff_tables.inc [master:t1,slave:t1]
+[connection master]
+DROP TABLE t1;
+DROP TABLE t2;
+include/rpl_end.inc