PS-4805: Re-implement compression dictionaries in 8.0 - v1

1.  Re-implement CREATE COMPRESSION_DICTIONARY, DROP COMPRESSION_DICTIONARY
2.  Allow CREATE TABLE to use 'WITH COMPRESSION_DICTIONARY'
3.  DDLs on tables with compression dictionary
4.  Prevent dropping of COMPRESSION DICTIONARY with existing references
5.  Allow compression_dictionary on tables with partitions
6.  Fix mysqldump to show compression dictionary
7.  New I_S views on compression dictionary tables
8.  Fix tests to use new I_S views
9.  Throw proper errors (ER_COMPRESSION_*)
10. Error handling in innodb_read_only mode
11. Handle upgrade from 5.7
12. Handle creation of dictionary tables in 8.0 at startup (mysql-8.0 -> PS8.0 )
13. Add upgrade test
14. Prevent direct access to mysql.compression_dictionary and mysql.compression_dictionary_cols table

client/mysqldump.cc

@@ -1004,6 +1004,10 @@ static int get_options(int *argc, char ***argv) {
 
   ignore_table =
       new collation_unordered_set<string>(charset_info, PSI_NOT_INSTRUMENTED);
+
+  processed_compression_dictionaries =
+      new collation_unordered_set<string>(charset_info, PSI_NOT_INSTRUMENTED);
+
   /* Don't copy internal log tables */
   ignore_table->insert("mysql.apply_status");
   ignore_table->insert("mysql.schema");
@@ -1509,6 +1513,12 @@ static void free_resources() {
     delete ignore_table;
     ignore_table = nullptr;
   }
+
+  if (processed_compression_dictionaries != nullptr) {
+    delete processed_compression_dictionaries;
+    processed_compression_dictionaries = nullptr;
+  }
+
   if (insert_pat_inited) dynstr_free(&insert_pat);
   if (opt_ignore_error) my_free(opt_ignore_error);
   my_end(my_end_arg);
@@ -2982,9 +2992,9 @@ static void print_optional_create_compression_dictionary(
   */
   if (!processed_compression_dictionaries->count(dictionary_name)) {
     static const constexpr char get_zip_dict_data_stmt[] =
-        "SELECT `ZIP_DICT` "
-        "FROM `INFORMATION_SCHEMA`.`XTRADB_ZIP_DICT` "
-        "WHERE `NAME` = '%s'";
+        "SELECT `DICT_DATA` "
+        "FROM `INFORMATION_SCHEMA`.`COMPRESSION_DICTIONARY` "
+        "WHERE `DICT_NAME` = '%s'";
 
     processed_compression_dictionaries->emplace(
         my_strdup(PSI_NOT_INSTRUMENTED, dictionary_name, MYF(0)));

mysql-test/r/dd_is_compatibility_cs.result

@@ -163,6 +163,8 @@ COLLATION_CHARACTER_SET_APPLICABILITY
 COLUMNS
 COLUMN_PRIVILEGES
 COLUMN_STATISTICS
+COMPRESSION_DICTIONARY
+COMPRESSION_DICTIONARY_TABLES
 ENGINES
 EVENTS
 FILES

mysql-test/r/dd_upgrade_compression_dict.result

@@ -0,0 +1,299 @@
+# Set different paths for --datadir
+# Copy the remote tablespace & DB zip files from suite location to working location.
+# Check that the file exists in the working folder.
+# Unzip the zip file.
+# Stop DB server which was created by MTR default
+# Start the 8.0 server on 5.7 datadir
+# restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=MYSQLD_DATADIR1
+# Execute mysql_upgrade
+mysql.columns_priv                                 OK
+mysql.component                                    OK
+mysql.db                                           OK
+mysql.default_roles                                OK
+mysql.engine_cost                                  OK
+mysql.func                                         OK
+mysql.general_log                                  OK
+mysql.global_grants                                OK
+mysql.gtid_executed                                OK
+mysql.help_category                                OK
+mysql.help_keyword                                 OK
+mysql.help_relation                                OK
+mysql.help_topic                                   OK
+mysql.innodb_index_stats                           OK
+mysql.innodb_table_stats                           OK
+mysql.ndb_binlog_index                             OK
+mysql.password_history                             OK
+mysql.plugin                                       OK
+mysql.procs_priv                                   OK
+mysql.proxies_priv                                 OK
+mysql.role_edges                                   OK
+mysql.server_cost                                  OK
+mysql.servers                                      OK
+mysql.slave_master_info                            OK
+mysql.slave_relay_log_info                         OK
+mysql.slave_worker_info                            OK
+mysql.slow_log                                     OK
+mysql.tables_priv                                  OK
+mysql.time_zone                                    OK
+mysql.time_zone_leap_second                        OK
+mysql.time_zone_name                               OK
+mysql.time_zone_transition                         OK
+mysql.time_zone_transition_type                    OK
+mysql.user                                         OK
+mtr.global_suppressions                            OK
+mtr.test_suppressions                              OK
+sys.sys_config                                     OK
+test.t1                                            OK
+test.t10                                           OK
+test.t11                                           OK
+test.t2                                            OK
+test.t3                                            OK
+test.t4                                            OK
+test.t5                                            OK
+test.t6                                            OK
+test.t7                                            OK
+test.t8                                            OK
+test.t9                                            OK
+SHOW CREATE TABLE test.t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` text /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd` */
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SHOW CREATE TABLE test.t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `a` text /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `d1` */
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SHOW CREATE TABLE test.t3;
+Table	Create Table
+t3	CREATE TABLE `t3` (
+  `a` text /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `d2` */
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SHOW CREATE TABLE test.t4;
+Table	Create Table
+t4	CREATE TABLE `t4` (
+  `a` int(11) NOT NULL AUTO_INCREMENT,
+  `b` text /*!50633 COLUMN_FORMAT COMPRESSED */,
+  `c` varchar(1000) /*!50633 COLUMN_FORMAT COMPRESSED */ DEFAULT NULL,
+  `d` varchar(10000) /*!50633 COLUMN_FORMAT COMPRESSED */ DEFAULT 'hello, world! see you!',
+  PRIMARY KEY (`a`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1
+SHOW CREATE TABLE test.t5;
+Table	Create Table
+t5	CREATE TABLE `t5` (
+  `id` bigint(20) unsigned NOT NULL,
+  `a` varchar(250) /*!50633 COLUMN_FORMAT COMPRESSED */ DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY RANGE (`id`)
+(PARTITION p1 VALUES LESS THAN (100) ENGINE = InnoDB,
+ PARTITION p2 VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */
+SHOW CREATE TABLE test.t6;
+Table	Create Table
+t6	CREATE TABLE `t6` (
+  `id` bigint(20) unsigned NOT NULL,
+  `a` varchar(250) /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `numbers` */ DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY RANGE (`id`)
+(PARTITION p1 VALUES LESS THAN (100) ENGINE = InnoDB,
+ PARTITION p2 VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */
+SHOW CREATE TABLE test.t7;
+Table	Create Table
+t7	CREATE TABLE `t7` (
+  `id` bigint(20) unsigned NOT NULL,
+  `a` varchar(250) /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `numbers` */ DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY HASH (`id`)
+PARTITIONS 2 */
+SHOW CREATE TABLE test.t8;
+Table	Create Table
+t8	CREATE TABLE `t8` (
+  `id` bigint(20) unsigned NOT NULL,
+  `a` varchar(250) /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `numbers` */ DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY RANGE (`id`)
+SUBPARTITION BY HASH (`id`)
+(PARTITION p0 VALUES LESS THAN (1024)
+ (SUBPARTITION s0 ENGINE = InnoDB,
+  SUBPARTITION s1 ENGINE = InnoDB),
+ PARTITION p1 VALUES LESS THAN MAXVALUE
+ (SUBPARTITION s2 ENGINE = InnoDB,
+  SUBPARTITION s3 ENGINE = InnoDB)) */
+SHOW CREATE TABLE test.t9;
+Table	Create Table
+t9	CREATE TABLE `t9` (
+  `id` bigint(20) unsigned NOT NULL,
+  `a` varchar(250) /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `numbers` */ DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY RANGE (`id`)
+SUBPARTITION BY HASH (`id`)
+SUBPARTITIONS 2
+(PARTITION p0 VALUES LESS THAN (1024) ENGINE = InnoDB,
+ PARTITION p1 VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */
+SHOW CREATE TABLE test.t10;
+Table	Create Table
+t10	CREATE TABLE `t10` (
+  `id` int(11) NOT NULL,
+  `a` blob NOT NULL /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `numbers` */,
+  `vchar` char(2) GENERATED ALWAYS AS (substr(`a`,2,2)) VIRTUAL,
+  KEY `vchar` (`vchar`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SHOW CREATE TABLE test.t11;
+Table	Create Table
+t11	CREATE TABLE `t11` (
+  `f1` int(11) NOT NULL,
+  `f2` blob /*!50633 COLUMN_FORMAT COMPRESSED */,
+  `f3` blob GENERATED ALWAYS AS (`f2`) VIRTUAL,
+  PRIMARY KEY (`f1`),
+  KEY `f3` (`f3`(200))
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SET @long_dictionary_data = REPEAT('ab', 32506 / 2);
+SELECT dict_data = @long_dictionary_data AS long_dictionary_data_matches FROM information_schema.compression_dictionary WHERE dict_name = 'd1';
+long_dictionary_data_matches
+1
+SET @dict_data8 = 'aaaaaaaabbbbbbbbccccccccdddddddd';
+SELECT dict_data = @dict_data8 AS short_dictionary_data_matches FROM information_schema.compression_dictionary WHERE dict_name = 'd2';
+short_dictionary_data_matches
+1
+# Retrieve all compression dictionaries
+# Long dictionary data is already retrieved and verifed above
+SELECT * FROM information_schema.compression_dictionary where dict_name != 'd1';
+DICT_VERSION	DICT_NAME	DICT_DATA
+1	d2	aaaaaaaabbbbbbbbccccccccdddddddd
+1	dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd	abcd
+1	numbers	onetwothree
+# Retrieve all data from tables created in 5.7
+SELECT LENGTH(a) FROM t1;
+LENGTH(a)
+160
+SELECT LENGTH(a) FROM t2;
+LENGTH(a)
+65012
+SELECT LENGTH(a) FROM t3;
+LENGTH(a)
+128
+SELECT a, LENGTH(b), LENGTH(c), LENGTH(d) FROM t4;
+a	LENGTH(b)	LENGTH(c)	LENGTH(d)
+1	1200	80	22
+2	1500	40	5500
+SELECT id, LENGTH(a) FROM t5;
+id	LENGTH(a)
+1	180
+2000	90
+SELECT id, LENGTH(a) FROM t6;
+id	LENGTH(a)
+1	240
+200	240
+300	250
+SELECT id, LENGTH(a) FROM t7;
+id	LENGTH(a)
+200	240
+300	250
+1	240
+SELECT id, LENGTH(a) FROM t8;
+id	LENGTH(a)
+200	240
+300	250
+1	240
+SELECT id, LENGTH(a) FROM t9;
+id	LENGTH(a)
+200	240
+300	250
+1	240
+SELECT id, LENGTH(a) FROM t10;
+id	LENGTH(a)
+1	8000
+SELECT f1, LENGTH(f2), LENGTH(f3) FROM t11;
+f1	LENGTH(f2)	LENGTH(f3)
+1	96	96
+# Do ALTER before restart and check schema
+ALTER TABLE t1 ADD COLUMN b BLOB COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `d2`;
+SET @t1_data = REPEAT('aaaaaaaabbbbbbbbccccccccdddddddd', 40);
+INSERT INTO t1 VALUES(@t1_data, @t1_data);
+SELECT LENGTH(a), LENGTH(b) FROM t1;
+LENGTH(a)	LENGTH(b)
+160	NULL
+1280	1280
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` text /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd` */,
+  `b` blob /*!50633 COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `d2` */
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+# restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=MYSQLD_DATADIR1
+# Do ALTER before restart and check schema
+ALTER TABLE t9 ADD COLUMN b BLOB COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY `d2`;
+INSERT INTO t9 VALUES(240, @t1_data, @t1_data);
+SELECT id, LENGTH(a), LENGTH(b) FROM t9;
+id	LENGTH(a)	LENGTH(b)
+200	240	NULL
+300	250	NULL
+240	NULL	NULL
+1	240	NULL
+# Start the 8.0 server on 5.7 datadir
+SET GLOBAL innodb_fast_shutdown = 0;
+# restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=MYSQLD_DATADIR1 --innodb-read-only
+SET @long_dictionary_data = REPEAT('ab', 32506 / 2);
+SELECT dict_data = @long_dictionary_data AS long_dictionary_data_matches FROM information_schema.compression_dictionary WHERE dict_name = 'd1';
+long_dictionary_data_matches
+1
+SET @dict_data8 = 'aaaaaaaabbbbbbbbccccccccdddddddd';
+SELECT dict_data = @dict_data8 AS short_dictionary_data_matches FROM information_schema.compression_dictionary WHERE dict_name = 'd2';
+short_dictionary_data_matches
+1
+# Retrieve all compression dictionaries
+# Long dictionary data is already retrieved and verifed above
+SELECT * FROM information_schema.compression_dictionary where dict_name != 'd1';
+DICT_VERSION	DICT_NAME	DICT_DATA
+1	d2	aaaaaaaabbbbbbbbccccccccdddddddd
+1	dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd	abcd
+1	numbers	onetwothree
+# Retrieve all data from tables created in 5.7
+SELECT LENGTH(a) FROM t1;
+LENGTH(a)
+160
+1280
+SELECT LENGTH(a) FROM t2;
+LENGTH(a)
+65012
+SELECT LENGTH(a) FROM t3;
+LENGTH(a)
+128
+SELECT a, LENGTH(b), LENGTH(c), LENGTH(d) FROM t4;
+a	LENGTH(b)	LENGTH(c)	LENGTH(d)
+1	1200	80	22
+2	1500	40	5500
+SELECT id, LENGTH(a) FROM t5;
+id	LENGTH(a)
+1	180
+2000	90
+SELECT id, LENGTH(a) FROM t6;
+id	LENGTH(a)
+1	240
+200	240
+300	250
+SELECT id, LENGTH(a) FROM t7;
+id	LENGTH(a)
+200	240
+300	250
+1	240
+SELECT id, LENGTH(a) FROM t8;
+id	LENGTH(a)
+200	240
+300	250
+1	240
+SELECT id, LENGTH(a) FROM t9;
+id	LENGTH(a)
+200	240
+300	250
+240	NULL
+1	240
+SELECT id, LENGTH(a) FROM t10;
+id	LENGTH(a)
+1	8000
+SELECT f1, LENGTH(f2), LENGTH(f3) FROM t11;
+f1	LENGTH(f2)	LENGTH(f3)
+1	96	96
+# Stop the server
+# Remove copied files
+# Restart the server with default options.
+# restart

mysql-test/r/dd_upgrade_compression_dict_80.result

@@ -0,0 +1,20 @@
+# create bootstrap file
+# Stop the MTR default DB server
+# Run the bootstrap command which skips creation of compression dictionary tables
+# restart: --datadir=MYSQLD_DATADIR --innodb_log_file_size=LOG_FILE_SIZE --innodb_log_files_in_group=2 --innodb_undo_tablespaces=2
+CREATE COMPRESSION_DICTIONARY d1('blah');
+CREATE TABLE t1(a BLOB COLUMN_FORMAT COMPRESSED WITH COMPRESSION_DICTIONARY d1);
+SELECT * FROM INFORMATION_SCHEMA.COMPRESSION_DICTIONARY;
+DICT_VERSION	DICT_NAME	DICT_DATA
+1	d1	blah
+SELECT * FROM INFORMATION_SCHEMA.COMPRESSION_DICTIONARY_TABLES;
+TABLE_SCHEMA	TABLE_NAME	COLUMN_NAME	DICT_NAME
+test	t1	a	d1
+DROP TABLE t1;
+DROP COMPRESSION_DICTIONARY d1;
+SELECT * FROM INFORMATION_SCHEMA.COMPRESSION_DICTIONARY;
+DICT_VERSION	DICT_NAME	DICT_DATA
+SELECT * FROM INFORMATION_SCHEMA.COMPRESSION_DICTIONARY_TABLES;
+TABLE_SCHEMA	TABLE_NAME	COLUMN_NAME	DICT_NAME
+# Shutdown the server
+# restart