BUG#28720023 BINLOG_ENCRYPTION IS NOT WORKING WITH WOLFSSL

Problem
-------

Binlog encryption feature was designed considering the following OpenSSL
library functions behaviors:
- EVP_BytesToKey function supports EVP_sha512;
- EVP_EncryptUpdate function with padding always encrypts the whole
  source buffer keeping cipher context;
- EVP_DecryptUpdate function with padding always decrypts the whole
  source buffer keeping cipher context and allows decrypting content
  to the same buffer as the source content;

In current WolfSSL library version supported by MySQL 8.0.14 (3.14), the
above mentioned functions behavior is not the same as OpenSSL's.

This patch adapts the Rpl_cipher functions to WolfSSL functions
behavior.

Author: Joao Gramacho

cmake/ssl.cmake

@@ -105,6 +105,7 @@ MACRO (MYSQL_USE_WOLFSSL)
     -DOPENSSL_EXTRA
     -DSESSION_CERT
     -DWC_NO_HARDEN
+    -DWOLFSSL_AES_COUNTER
     -DWOLFSSL_AES_DIRECT
     -DWOLFSSL_ALLOW_TLSV10
     -DWOLFSSL_CERT_EXT

share/errmsg-utf8.txt

@@ -8736,6 +8736,9 @@ ER_GRP_TRX_CONSISTENCY_BEGIN_NOT_ALLOWED
 ER_FUNCTIONAL_INDEX_ROW_VALUE_IS_NOT_ALLOWED
   eng "Expression of functional index '%s' cannot refer to a row value."
 
+ER_RPL_ENCRYPTION_FAILED_TO_ENCRYPT
+  eng "Failed to encrypt content to write into binlog file: %s."
+
 #
 #  End of 8.0 error messages.
 #
@@ -18740,6 +18743,9 @@ ER_GRP_RPL_GTID_SET_EXTRACT_ERROR_DURING_RECOVERY
 ER_SECONDARY_ENGINE_PLUGIN
   eng "%s"
 
+ER_SERVER_RPL_ENCRYPTION_FAILED_TO_ENCRYPT
+  eng "Failed to encrypt content to write into binlog file: %s."
+
 #
 # End of 8.0 error messages intended to be logged to the server error log.
 #

sql/binlog.cc

@@ -3606,9 +3606,9 @@ bool MYSQL_BIN_LOG::open(PSI_file_key log_file_key, const char *log_name,
 err:
   if (binlog_error_action == ABORT_SERVER) {
     exec_binlog_error_action_abort(
-        "Either disk is full or file system is read "
-        "only while opening the binlog. Aborting the"
-        " server.");
+        "Either disk is full, file system is read only or "
+        "there was an encryption error while opening the binlog. "
+        "Aborting the server.");
   } else
     LogErr(ERROR_LEVEL, ER_BINLOG_CANT_OPEN_FOR_LOGGING, name, errno);
 
@@ -4901,9 +4901,9 @@ bool MYSQL_BIN_LOG::open_binlog(
   close_purge_index_file();
   if (binlog_error_action == ABORT_SERVER) {
     exec_binlog_error_action_abort(
-        "Either disk is full or file system is read "
-        "only while opening the binlog. Aborting the"
-        " server.");
+        "Either disk is full, file system is read only or "
+        "there was an encryption error while opening the binlog. "
+        "Aborting the server.");
   } else {
     LogErr(ERROR_LEVEL, ER_BINLOG_CANT_USE_FOR_LOGGING,
            (new_name) ? new_name : name, errno);

sql/binlog_istream.cc

@@ -65,6 +65,8 @@ const char *Binlog_read_error::get_str() const {
              "please check if keyring plugin is loaded";
     case READ_ENCRYPTED_LOG_FILE_IS_NOT_SUPPORTED:
       return "Reading encrypted log files directly is not supported.";
+    case ERROR_DECRYPTING_FILE:
+      return "Failed to decrypt content read from binlog file.";
     default:
       /* There must be something wrong in the code if it reaches this branch. */
       DBUG_ASSERT(0);
@@ -95,7 +97,8 @@ bool Binlog_encryption_istream::open(
   m_decryptor = encryption_header->get_decryptor();
   if (m_decryptor->open(password, encryption_header->get_header_size())) {
     m_decryptor.reset(nullptr);
-    DBUG_RETURN(true);
+    DBUG_RETURN(
+        binlog_read_error->set_type(Binlog_read_error::ERROR_DECRYPTING_FILE));
   }
 
   DBUG_RETURN(false);

sql/binlog_istream.h

@@ -69,7 +69,8 @@ class Binlog_read_error {
     BAD_BINLOG_MAGIC,
     INVALID_ENCRYPTION_HEADER,
     CANNOT_GET_FILE_PASSWORD,
-    READ_ENCRYPTED_LOG_FILE_IS_NOT_SUPPORTED
+    READ_ENCRYPTED_LOG_FILE_IS_NOT_SUPPORTED,
+    ERROR_DECRYPTING_FILE
   };
 
   Binlog_read_error() {}

sql/binlog_ostream.cc

@@ -149,14 +149,27 @@ Binlog_cache_storage::~Binlog_cache_storage() { close(); }
 
 Binlog_encryption_ostream::~Binlog_encryption_ostream() { close(); }
 
+#define THROW_RPL_ENCRYPTION_FAILED_TO_ENCRYPT_ERROR                        \
+  char err_msg[MYSQL_ERRMSG_SIZE];                                          \
+  ERR_error_string_n(ERR_get_error(), err_msg, MYSQL_ERRMSG_SIZE);          \
+  LogErr(ERROR_LEVEL, ER_SERVER_RPL_ENCRYPTION_FAILED_TO_ENCRYPT, err_msg); \
+  if (current_thd) {                                                        \
+    if (current_thd->is_error()) current_thd->clear_error();                \
+    my_error(ER_RPL_ENCRYPTION_FAILED_TO_ENCRYPT, MYF(0), err_msg);         \
+  }
+
 bool Binlog_encryption_ostream::open(
     std::unique_ptr<Truncatable_ostream> down_ostream) {
   DBUG_ASSERT(down_ostream != nullptr);
   m_header = Rpl_encryption_header::get_new_default_header();
   const Key_string password_str = m_header->generate_new_file_password();
   m_encryptor.reset(nullptr);
   m_encryptor = m_header->get_encryptor();
-  if (m_encryptor->open(password_str, m_header->get_header_size())) return true;
+  if (m_encryptor->open(password_str, m_header->get_header_size())) {
+    THROW_RPL_ENCRYPTION_FAILED_TO_ENCRYPT_ERROR;
+    m_encryptor.reset(nullptr);
+    return true;
+  }
   m_down_ostream = std::move(down_ostream);
   return m_header->serialize(m_down_ostream.get());
 }
@@ -170,8 +183,12 @@ bool Binlog_encryption_ostream::open(
   m_header = std::move(header);
   m_encryptor.reset(nullptr);
   m_encryptor = m_header->get_encryptor();
-  m_encryptor->open(m_header->decrypt_file_password(),
-                    m_header->get_header_size());
+  if (m_encryptor->open(m_header->decrypt_file_password(),
+                        m_header->get_header_size())) {
+    THROW_RPL_ENCRYPTION_FAILED_TO_ENCRYPT_ERROR;
+    m_encryptor.reset(nullptr);
+    return true;
+  }
 
   return seek(0);
 }
@@ -196,7 +213,11 @@ bool Binlog_encryption_ostream::write(const unsigned char *buffer,
     int encrypt_len =
         std::min(length, static_cast<my_off_t>(ENCRYPT_BUFFER_SIZE));
 
-    if (m_encryptor->encrypt(encrypt_buffer, ptr, encrypt_len)) return true;
+    if (m_encryptor->encrypt(encrypt_buffer, ptr, encrypt_len)) {
+      THROW_RPL_ENCRYPTION_FAILED_TO_ENCRYPT_ERROR;
+      return true;
+    }
+
     if (m_down_ostream->write(encrypt_buffer, encrypt_len)) return true;
 
     ptr += encrypt_len;

sql/rpl_cipher.cc

@@ -17,14 +17,45 @@
 #include <algorithm>
 #include "my_byteorder.h"
 
+#define HAVE_BYTESTOKEY_SHA512_HANDLING
+#define HAVE_DECRYPTION_INTO_SAME_SOURCE_BUFFER
+
+#ifdef HAVE_WOLFSSL
+#undef HAVE_BYTESTOKEY_SHA512_HANDLING
+#undef HAVE_DECRYPTION_INTO_SAME_SOURCE_BUFFER
+#endif
+
 int Rpl_cipher::get_header_size() { return m_header_size; }
 
+std::unique_ptr<Rpl_cipher> Aes_ctr::get_encryptor() {
+  std::unique_ptr<Rpl_cipher> cipher(new Aes_ctr_encryptor);
+  return cipher;
+}
+
+std::unique_ptr<Rpl_cipher> Aes_ctr::get_decryptor() {
+  std::unique_ptr<Rpl_cipher> cipher(new Aes_ctr_decryptor);
+  return cipher;
+}
+
 template <Cipher_type TYPE>
 bool Aes_ctr_cipher<TYPE>::open(const Key_string &password, int header_size) {
   m_header_size = header_size;
-  if (EVP_BytesToKey(EVP_aes_256_ctr(), EVP_sha512(), NULL, password.data(),
-                     password.length(), 1, m_file_key, m_iv) == 0)
+#ifdef HAVE_BYTESTOKEY_SHA512_HANDLING
+  if (EVP_BytesToKey(Aes_ctr::get_evp_cipher(), Aes_ctr::get_evp_md(), NULL,
+                     password.data(), password.length(), 1, m_file_key,
+                     m_iv) == 0)
     return true;
+#else
+  unsigned char buffer[64];
+  unsigned int hash_size = 0;
+  EVP_MD_CTX *ctx = EVP_MD_CTX_create();
+  EVP_DigestInit_ex(ctx, Aes_ctr::get_evp_md(), nullptr);
+  EVP_DigestUpdate(ctx, password.data(), password.length());
+  EVP_DigestFinal_ex(ctx, buffer, &hash_size);
+  EVP_MD_CTX_destroy(ctx);
+  memcpy(m_file_key, buffer, FILE_KEY_LENGTH);
+  memcpy(m_iv, buffer + FILE_KEY_LENGTH, AES_BLOCK_SIZE);
+#endif
 
   /*
     AES-CTR counter is set to 0. Data stream is always encrypted beginning with
@@ -88,11 +119,11 @@ bool Aes_ctr_cipher<TYPE>::init_cipher(uint64_t offset) {
   std::swap(m_iv[11], m_iv[12]);
 
   int res;
-  /* EVP_EncryptInit_ex() return 1 for success and 0 for failure */
-  if (TYPE == Cipher_type::ENCRYPT)
-    res = EVP_EncryptInit_ex(m_ctx, EVP_aes_256_ctr(), NULL, m_file_key, m_iv);
-  else
-    res = EVP_DecryptInit_ex(m_ctx, EVP_aes_256_ctr(), NULL, m_file_key, m_iv);
+  const EVP_CIPHER *cipher_type = Aes_ctr::get_evp_cipher();
+
+  /* EVP_CipherInit() returns 1 for success and 0 for failure */
+  res = EVP_CipherInit(m_ctx, cipher_type, m_file_key, m_iv, TYPE);
+
   DBUG_RETURN(res == 0);
 }
 
@@ -105,34 +136,57 @@ void Aes_ctr_cipher<TYPE>::deinit_cipher() {
 template <Cipher_type TYPE>
 bool Aes_ctr_cipher<TYPE>::encrypt(unsigned char *dest,
                                    const unsigned char *src, int length) {
-  int out_len = 0;
+  DBUG_ENTER("Aes_ctr_cipher::encrypt");
 
   if (TYPE == Cipher_type::DECRYPT) {
     /* It should never be called by a decrypt cipher */
     DBUG_ASSERT(0);
-    return true;
+    DBUG_RETURN(true);
   }
 
-  if (EVP_EncryptUpdate(m_ctx, dest, &out_len, src, length) == 0) return true;
+  /* length == 0 : nothing to encrypt */
+  if (length == 0) DBUG_RETURN(false);
+
+  if (EVP_Cipher(m_ctx, dest, (unsigned char *)src, length) == 0)
+    DBUG_RETURN(true);
 
-  DBUG_ASSERT(out_len == length);
-  return false;
+  DBUG_RETURN(false);
 }
 
 template <Cipher_type TYPE>
 bool Aes_ctr_cipher<TYPE>::decrypt(unsigned char *dest,
                                    const unsigned char *src, int length) {
-  int out_len = 0;
+  DBUG_ENTER("Aes_ctr_cipher::decrypt");
 
   if (TYPE == Cipher_type::ENCRYPT) {
     /* It should never be called by an encrypt cipher */
     DBUG_ASSERT(0);
-    return true;
+    DBUG_RETURN(true);
+  }
+
+  /* length == 0 : nothing to decrypt */
+  if (length == 0) DBUG_RETURN(false);
+
+#ifdef HAVE_DECRYPTION_INTO_SAME_SOURCE_BUFFER
+  if (EVP_Cipher(m_ctx, dest, (unsigned char *)src, length) == 0)
+    DBUG_RETURN(true);
+#else
+  const int DECRYPTED_BUFFER_SIZE = AES_BLOCK_SIZE * 2048;
+  unsigned char buffer[DECRYPTED_BUFFER_SIZE];
+
+  /* Decrypt in up to DECRYPTED_BUFFER_SIZE chunks */
+  while (length != 0) {
+    int chunk_len = std::min(length, DECRYPTED_BUFFER_SIZE);
+    if (EVP_Cipher(m_ctx, buffer, (unsigned char *)src, chunk_len) == 0)
+      DBUG_RETURN(true);
+    memcpy(dest, buffer, chunk_len);
+    src += chunk_len;
+    dest += chunk_len;
+    length -= chunk_len;
   }
+#endif
 
-  if (EVP_DecryptUpdate(m_ctx, dest, &out_len, src, length) == 0) return true;
-  DBUG_ASSERT(out_len == length);
-  return false;
+  DBUG_RETURN(false);
 }
 
 template class Aes_ctr_cipher<Cipher_type::ENCRYPT>;

sql/rpl_cipher.h

@@ -17,6 +17,7 @@
 #define RPL_CIPHER_INCLUDED
 
 #include <openssl/evp.h>
+#include <memory>
 #include <string>
 
 /**
@@ -134,6 +135,42 @@ class Rpl_cipher {
   int m_header_size = 0;
 };
 
+/**
+  @class Aes_ctr
+
+  The class provides standards to be used by the Aes_ctr ciphers.
+*/
+class Aes_ctr {
+ public:
+  static const int PASSWORD_LENGTH = 32;
+  static const int AES_BLOCK_SIZE = 16;
+  static const int FILE_KEY_LENGTH = 32;
+  /**
+    Returns the message digest function to be uses when opening the cipher.
+
+    @return SHA-512 message digest.
+  */
+  static const EVP_MD *get_evp_md() { return EVP_sha512(); };
+  /**
+    Returns the cipher to be uses when using the cipher.
+
+    @return AES-256-CTR.
+  */
+  static const EVP_CIPHER *get_evp_cipher() { return EVP_aes_256_ctr(); };
+  /**
+    Returns a new unique Rpl_cipher encryptor.
+
+    @return A new Rpl_cipher encryptor.
+  */
+  static std::unique_ptr<Rpl_cipher> get_encryptor();
+  /**
+    Returns a new unique Rpl_cipher decryptor.
+
+    @return A new Rpl_cipher decryptor.
+  */
+  static std::unique_ptr<Rpl_cipher> get_decryptor();
+};
+
 enum Cipher_type { ENCRYPT, DECRYPT };
 
 /**
@@ -145,9 +182,9 @@ enum Cipher_type { ENCRYPT, DECRYPT };
 template <Cipher_type TYPE>
 class Aes_ctr_cipher : public Rpl_cipher {
  public:
-  static const int PASSWORD_LENGTH = 32;
-  static const int AES_BLOCK_SIZE = 16;
-  static const int FILE_KEY_LENGTH = 32;
+  static const int PASSWORD_LENGTH = Aes_ctr::PASSWORD_LENGTH;
+  static const int AES_BLOCK_SIZE = Aes_ctr::AES_BLOCK_SIZE;
+  static const int FILE_KEY_LENGTH = Aes_ctr::FILE_KEY_LENGTH;
 
   virtual ~Aes_ctr_cipher();