WL#12475: Protocol Changes to specify compression configuration for connections

WL12475 provides a framework to configure compression algorithm and compression
level to be used with mysql protocol. Different compression algorithms
supported for mysql protocol compression are zlib and zstd.

MYSQL_OPT_COMPRESSION_ALGORITHMS option can be used to configure compression
algorithm from a client application. MYSQL_OPT_ZSTD_COMPRESSION_LEVEL option
can be used to specify compression level for zstd compression algorithm.

This WL also introduces a new server variable "protocol-compression-algorithms"
which can be used to configure servers compression algorithms. Based on how
this variable is configured mysql protocol compression algorithm is negotiated
between client and server accordingly.

This WL also extends CHANGE MASTER TO statement to allow users to configure
compression algorithm/level on slave replication channels. To view what
compression algorithm and compression level are configured on a connection
user can query status variables like compression_algorithm and compression_level.

Reviewed-by: Georgi Kodinov <georgi.kodinov@oracle.com>
             Gleb Shchepa <gleb.shchepa@oracle.com>
             Sven Sandberg <sven.sandberg@oracle.com>
             Nuno Carvalho <nuno.carvalho@oracle.com>
RB: 21705

Author: Bharathy Satish

client/base/mysql_connection_options.cc

@@ -32,6 +32,7 @@
 #include "caching_sha2_passwordopt-vars.h"
 #include "client/base/abstract_options_provider.h"
 #include "client/base/abstract_program.h"
+#include "compression.h"
 #include "m_ctype.h"
 #include "mysys_err.h"
 #include "typelib.h"
@@ -67,6 +68,18 @@ void Mysql_connection_options::create_options() {
   this->create_new_option(&this->m_compress, "compress",
                           "Use compression in server/client protocol.")
       ->set_short_character('C');
+  this->create_new_option(
+      &this->m_compress_algorithm, "compression-algorithms",
+      "Use compression algorithm in server/client protocol. Valid values "
+      "are any combination of 'zstd','zlib','uncompressed'");
+  this->create_new_option(
+          &this->m_zstd_compress_level, "zstd-compression-level",
+          "Use this compression level in the client/server protocol, in case "
+          "--compression-algorithms=zstd. Valid range is between 1 and 22, "
+          "inclusive. Default is 3.")
+      ->set_minimum_value(1)
+      ->set_maximum_value(22)
+      ->set_value(default_zstd_compression_level);
   this->create_new_option(&this->m_default_charset, "default-character-set",
                           "Set the default character set.")
       ->set_value("UTF8MB4");
@@ -131,6 +144,13 @@ MYSQL *Mysql_connection_options::create_connection() {
   MYSQL *connection = mysql_init(NULL);
   if (this->m_compress) mysql_options(connection, MYSQL_OPT_COMPRESS, NullS);
 
+  if (this->m_compress_algorithm.has_value())
+    mysql_options(connection, MYSQL_OPT_COMPRESSION_ALGORITHMS,
+                  this->m_compress_algorithm.value().c_str());
+
+  mysql_options(connection, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL,
+                &this->m_zstd_compress_level);
+
   if (this->m_ssl_options_provider.apply_for_connection(connection))
     return NULL;
 

client/base/mysql_connection_options.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -143,6 +143,8 @@ class Mysql_connection_options : public Composite_options_provider,
   Nullable<std::string> m_default_charset;
   Nullable<std::string> m_server_public_key;
   bool m_get_server_public_key;
+  uint m_zstd_compress_level;
+  Nullable<std::string> m_compress_algorithm;
 };
 
 }  // namespace Options

client/check/mysqlcheck.cc

@@ -30,6 +30,7 @@
 
 #include "caching_sha2_passwordopt-vars.h"
 #include "client/client_priv.h"
+#include "compression.h"
 #include "m_ctype.h"
 #include "my_alloc.h"
 #include "my_dbug.h"
@@ -69,6 +70,8 @@ static const char *default_charset = nullptr;
 static char *opt_plugin_dir = 0, *opt_default_auth = 0;
 static int first_error = 0;
 static const char *opt_skip_database = "";
+static uint opt_zstd_compress_level = default_zstd_compression_level;
+static char *opt_compress_algorithm = nullptr;
 #if defined(_WIN32)
 static char *shared_memory_base_name = 0;
 #endif
@@ -226,6 +229,17 @@ static struct my_option my_long_options[] = {
      GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
     {"version", 'V', "Output version information and exit.", 0, 0, 0,
      GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+    {"compression-algorithms", 0,
+     "Use compression algorithm in server/client protocol. Valid values "
+     "are any combination of 'zstd','zlib','uncompressed'.",
+     &opt_compress_algorithm, &opt_compress_algorithm, 0, GET_STR, REQUIRED_ARG,
+     0, 0, 0, 0, 0, 0},
+    {"zstd-compression-level", 0,
+     "Use this compression level in the client/server protocol, in case "
+     "--compression-algorithms=zstd. Valid range is between 1 and 22, "
+     "inclusive. Default is 3.",
+     &opt_zstd_compress_level, &opt_zstd_compress_level, 0, GET_UINT,
+     REQUIRED_ARG, 3, 1, 22, 0, 0, 0},
     {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
 
 static const char *load_default_groups[] = {"mysqlcheck", "client", 0};
@@ -431,6 +445,13 @@ static int dbConnect(char *host, char *user, char *passwd) {
   }
   mysql_init(&mysql_connection);
   if (opt_compress) mysql_options(&mysql_connection, MYSQL_OPT_COMPRESS, NullS);
+  if (opt_compress_algorithm)
+    mysql_options(&mysql_connection, MYSQL_OPT_COMPRESSION_ALGORITHMS,
+                  opt_compress_algorithm);
+
+  mysql_options(&mysql_connection, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL,
+                &opt_zstd_compress_level);
+
   if (SSL_SET_OPTIONS(&mysql_connection)) {
     fprintf(stderr, "%s", SSL_SET_OPTIONS_ERROR);
     return 1;

client/mysql.cc

@@ -40,6 +40,7 @@
 #include "client/client_priv.h"
 #include "client/my_readline.h"
 #include "client/pattern_matcher.h"
+#include "compression.h"
 #include "lex_string.h"
 #include "m_ctype.h"
 #include "my_compiler.h"
@@ -212,6 +213,8 @@ static uint prompt_counter;
 static char delimiter[16] = DEFAULT_DELIMITER;
 static size_t delimiter_length = 1;
 unsigned short terminal_width = 80;
+static uint opt_zstd_compress_level = default_zstd_compression_level;
+static char *opt_compress_algorithm = nullptr;
 
 #if defined(_WIN32)
 static char *shared_memory_base_name = 0;
@@ -1857,6 +1860,18 @@ static struct my_option my_long_options[] = {
      &opt_network_namespace, &opt_network_namespace, 0, GET_STR, REQUIRED_ARG,
      0, 0, 0, 0, 0, 0},
 #endif
+    {"compression-algorithms", 0,
+     "Use compression algorithm in server/client protocol. Valid values "
+     "are any combination of 'zstd','zlib','uncompressed'.",
+     &opt_compress_algorithm, &opt_compress_algorithm, 0, GET_STR, REQUIRED_ARG,
+     0, 0, 0, 0, 0, 0},
+    {"zstd-compression-level", 0,
+     "Use this compression level in the client/server protocol, in case "
+     "--compression-algorithms=zstd. Valid range is between 1 and 22, "
+     "inclusive. Default is 3.",
+     &opt_zstd_compress_level, &opt_zstd_compress_level, 0, GET_UINT,
+     REQUIRED_ARG, 3, 1, 22, 0, 0, 0},
+
     {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
 
 static void usage(int version) {
@@ -4533,6 +4548,12 @@ static bool init_connection_options(MYSQL *mysql) {
   if (opt_bind_addr) mysql_options(mysql, MYSQL_OPT_BIND, opt_bind_addr);
 
   if (opt_compress) mysql_options(mysql, MYSQL_OPT_COMPRESS, NullS);
+  if (opt_compress_algorithm)
+    mysql_options(mysql, MYSQL_OPT_COMPRESSION_ALGORITHMS,
+                  opt_compress_algorithm);
+
+  mysql_options(mysql, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL,
+                &opt_zstd_compress_level);
 
   if (using_opt_local_infile)
     mysql_options(mysql, MYSQL_OPT_LOCAL_INFILE, (char *)&opt_local_infile);

client/mysqladmin.cc

@@ -35,6 +35,7 @@
 #include <string>
 
 #include "client/client_priv.h"
+#include "compression.h"
 #include "m_ctype.h"
 #include "my_alloc.h"
 #include "my_compiler.h"
@@ -74,7 +75,8 @@ static char *opt_plugin_dir = 0, *opt_default_auth = 0;
 static uint opt_enable_cleartext_plugin = 0;
 static bool using_opt_enable_cleartext_plugin = 0;
 static bool opt_show_warnings = 0;
-
+static uint opt_zstd_compress_level = default_zstd_compression_level;
+static char *opt_compress_algorithm = nullptr;
 #if defined(_WIN32)
 static char *shared_memory_base_name = 0;
 #endif
@@ -287,6 +289,18 @@ static struct my_option my_long_options[] = {
     {"show_warnings", OPT_SHOW_WARNINGS, "Show warnings after execution",
      &opt_show_warnings, &opt_show_warnings, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0,
      0},
+    {"compression-algorithms", 0,
+     "Use compression algorithm in server/client protocol. Valid values "
+     "are any combination of 'zstd','zlib','uncompressed'.",
+     &opt_compress_algorithm, &opt_compress_algorithm, 0, GET_STR, REQUIRED_ARG,
+     0, 0, 0, 0, 0, 0},
+    {"zstd-compression-level", 0,
+     "Use this compression level in the client/server protocol, in case "
+     "--compression-algorithms=zstd. Valid range is between 1 and 22, "
+     "inclusive. Default is 3.",
+     &opt_zstd_compress_level, &opt_zstd_compress_level, 0, GET_UINT,
+     REQUIRED_ARG, 3, 1, 22, 0, 0, 0},
+
     {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
 
 static const char *load_default_groups[] = {"mysqladmin", "client", 0};
@@ -422,6 +436,13 @@ int main(int argc, char *argv[]) {
   mysql_options(&mysql, MYSQL_SET_CHARSET_NAME, default_charset);
   error_flags = (myf)(opt_nobeep ? 0 : ME_BELL);
 
+  if (opt_compress_algorithm)
+    mysql_options(&mysql, MYSQL_OPT_COMPRESSION_ALGORITHMS,
+                  opt_compress_algorithm);
+
+  mysql_options(&mysql, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL,
+                &opt_zstd_compress_level);
+
   if (opt_plugin_dir && *opt_plugin_dir)
     mysql_options(&mysql, MYSQL_PLUGIN_DIR, opt_plugin_dir);
 

client/mysqlbinlog.cc

@@ -46,6 +46,7 @@
 
 #include "caching_sha2_passwordopt-vars.h"
 #include "client/client_priv.h"
+#include "compression.h"
 #include "my_byteorder.h"
 #include "my_dbug.h"
 #include "my_default.h"
@@ -344,6 +345,8 @@ Sid_map *global_sid_map = nullptr;
 Checkable_rwlock *global_sid_lock = nullptr;
 Gtid_set *gtid_set_included = nullptr;
 Gtid_set *gtid_set_excluded = nullptr;
+static uint opt_zstd_compress_level = default_zstd_compression_level;
+static char *opt_compress_algorithm = nullptr;
 
 static bool opt_print_table_metadata;
 
@@ -1585,6 +1588,17 @@ static struct my_option my_long_options[] = {
      &opt_print_table_metadata, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
     {"compress", 'C', "Use compression in server/client protocol.",
      &opt_compress, &opt_compress, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+    {"compression-algorithms", 0,
+     "Use compression algorithm in server/client protocol. Valid values "
+     "are any combination of 'zstd','zlib','uncompressed'.",
+     &opt_compress_algorithm, &opt_compress_algorithm, 0, GET_STR, REQUIRED_ARG,
+     0, 0, 0, 0, 0, 0},
+    {"zstd-compression-level", 0,
+     "Use this compression level in the client/server protocol, in case "
+     "--compression-algorithms=zstd. Valid range is between 1 and 22, "
+     "inclusive. Default is 3.",
+     &opt_zstd_compress_level, &opt_zstd_compress_level, 0, GET_UINT,
+     REQUIRED_ARG, 3, 1, 22, 0, 0, 0},
     {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
 
 /**
@@ -1844,6 +1858,13 @@ static Exit_status safe_connect() {
   if (opt_plugin_dir && *opt_plugin_dir)
     mysql_options(mysql, MYSQL_PLUGIN_DIR, opt_plugin_dir);
 
+  if (opt_compress_algorithm)
+    mysql_options(mysql, MYSQL_OPT_COMPRESSION_ALGORITHMS,
+                  opt_compress_algorithm);
+
+  mysql_options(mysql, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL,
+                &opt_zstd_compress_level);
+
   if (opt_default_auth && *opt_default_auth)
     mysql_options(mysql, MYSQL_DEFAULT_AUTH, opt_default_auth);
 

client/mysqldump.cc

@@ -37,6 +37,7 @@
 #include <string>
 
 #include "client/client_priv.h"
+#include "compression.h"
 #include "m_ctype.h"
 #include "m_string.h"
 #include "map_helpers.h"
@@ -135,6 +136,10 @@ static bool server_supports_switching_charsets = true;
   otherwise backticks (``, non-standard MySQL feature).
 */
 static bool ansi_quotes_mode = false;
+
+static uint opt_zstd_compress_level = default_zstd_compression_level;
+static char *opt_compress_algorithm = nullptr;
+
 #define MYSQL_OPT_MASTER_DATA_EFFECTIVE_SQL 1
 #define MYSQL_OPT_MASTER_DATA_COMMENTED_SQL 2
 #define MYSQL_OPT_SLAVE_DATA_EFFECTIVE_SQL 1
@@ -562,6 +567,17 @@ static struct my_option my_long_options[] = {
      &opt_show_create_table_skip_secondary_engine,
      &opt_show_create_table_skip_secondary_engine, 0, GET_BOOL, NO_ARG, 0, 0, 0,
      0, 0, 0},
+    {"compression-algorithms", 0,
+     "Use compression algorithm in server/client protocol. Valid values "
+     "are any combination of 'zstd','zlib','uncompressed'.",
+     &opt_compress_algorithm, &opt_compress_algorithm, 0, GET_STR, REQUIRED_ARG,
+     0, 0, 0, 0, 0, 0},
+    {"zstd-compression-level", 0,
+     "Use this compression level in the client/server protocol, in case "
+     "--compression-algorithms=zstd. Valid range is between 1 and 22, "
+     "inclusive. Default is 3.",
+     &opt_zstd_compress_level, &opt_zstd_compress_level, 0, GET_UINT,
+     REQUIRED_ARG, 3, 1, 22, 0, 0, 0},
     {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
 
 static const char *load_default_groups[] = {"mysqldump", "client", 0};
@@ -1491,6 +1507,13 @@ static int connect_to_db(char *host, char *user, char *passwd) {
   set_server_public_key(&mysql_connection);
   set_get_server_public_key_option(&mysql_connection);
 
+  if (opt_compress_algorithm)
+    mysql_options(&mysql_connection, MYSQL_OPT_COMPRESSION_ALGORITHMS,
+                  opt_compress_algorithm);
+
+  mysql_options(&mysql_connection, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL,
+                &opt_zstd_compress_level);
+
   if (opt_network_timeout) {
     uint timeout = 700;
     ulong max_packet_allowed = 1024L * 1024L * 1024L;

client/mysqlimport.cc

@@ -33,6 +33,7 @@
 #include <time.h>
 
 #include "client/client_priv.h"
+#include "compression.h"
 #include "my_alloc.h"
 #include "my_dbug.h"
 #include "my_default.h"
@@ -76,6 +77,9 @@ static char *opt_bind_addr = NULL;
 static char *opt_mysql_unix_port = 0;
 static char *opt_plugin_dir = 0, *opt_default_auth = 0;
 static longlong opt_ignore_lines = -1;
+static uint opt_zstd_compress_level = default_zstd_compression_level;
+static char *opt_compress_algorithm = nullptr;
+
 #include "caching_sha2_passwordopt-vars.h"
 #include "sslopt-vars.h"
 
@@ -211,6 +215,17 @@ static struct my_option my_long_options[] = {
      0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
     {"version", 'V', "Output version information and exit.", 0, 0, 0,
      GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+    {"compression-algorithms", 0,
+     "Use compression algorithm in server/client protocol. Valid values "
+     "are any combination of 'zstd','zlib','uncompressed'.",
+     &opt_compress_algorithm, &opt_compress_algorithm, 0, GET_STR, REQUIRED_ARG,
+     0, 0, 0, 0, 0, 0},
+    {"zstd-compression-level", 0,
+     "Use this compression level in the client/server protocol, in case "
+     "--compression-algorithms=zstd. Valid range is between 1 and 22, "
+     "inclusive. Default is 3.",
+     &opt_zstd_compress_level, &opt_zstd_compress_level, 0, GET_UINT,
+     REQUIRED_ARG, 3, 1, 22, 0, 0, 0},
     {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
 
 static const char *load_default_groups[] = {"mysqlimport", "client", 0};
@@ -419,6 +434,14 @@ static MYSQL *db_connect(char *host, char *database, char *user, char *passwd) {
   } else if (!(mysql = mysql_init(NULL)))
     return 0;
   if (opt_compress) mysql_options(mysql, MYSQL_OPT_COMPRESS, NullS);
+
+  if (opt_compress_algorithm)
+    mysql_options(mysql, MYSQL_OPT_COMPRESSION_ALGORITHMS,
+                  opt_compress_algorithm);
+
+  mysql_options(mysql, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL,
+                &opt_zstd_compress_level);
+
   if (opt_local_file)
     mysql_options(mysql, MYSQL_OPT_LOCAL_INFILE, (char *)&opt_local_file);
   if (SSL_SET_OPTIONS(mysql)) {