Skip to content
Permalink
Browse files
MDEV-18323 Convert MySQL JSON type to MariaDB TEXT in mysql_upgrade 
This patch solves two key problems.
1. There is a type number clash between MySQL and MariaDB. The number
   245, used for MariaDB Virtual Fields is the same as MySQL's JSON.
   This leads to corrupt FRM errors if unhandled. The code properly
   checks frm table version number and if it matches 5.7+ (until 10.0+)
   it will assume it is dealing with a MySQL table with the JSON
   datatype.
2. MySQL JSON datatype uses a proprietary format to pack JSON data. The
   patch introduces a datatype plugin which parses the format and convers
   it to its string representation.

The intended conversion path is to only use the JSON datatype within
ALTER TABLE <table> FORCE, to force a table recreate. This happens
during mysql_upgrade or via a direct ALTER TABLE <table> FORCE.
  • Loading branch information
@cvicentiu
cvicentiu committed Oct 28, 2020
1 parent 85c686e commit f6549e9
Show file tree
Hide file tree
Showing 19 changed files with 1,224 additions and 2 deletions.
1 mysql-test/main/mysql_json_table_recreate.opt
View file
@@ -0,0 +1 @@
--plugin-load=$TYPE_MYSQL_JSON_SO
171 mysql-test/main/mysql_json_table_recreate.result
View file
@@ -0,0 +1,171 @@
#
# The following test takes 2 tables containing a JSON column and attempts
# to repair them.
#
# The tables header is (Description, Expected, Actual), where description
# shows a brief description what the JSON value is testing in the MariaDB
# implementation. Expected is the longtext string and actual is the JSON
# column that needs to be converted to MariaDB's representation of
# LONGTEXT.
#
call mtr.add_suppression("Table rebuild required");
call mtr.add_suppression("is marked as crashed");
call mtr.add_suppression("Checking");
SET NAMES utf8;
#
# Check that only ALTER TABLE ... FORCE is allowed on a MySQL 5.7 table
# with a JSON column.
#
show create table tempty;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.tempty` FORCE" or dump/reload to fix it!
select * from tempty;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.tempty` FORCE" or dump/reload to fix it!
alter table tempty force;
show create table tempty;
Table Create Table
tempty CREATE TABLE `tempty` (
`t` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1
show create table mysql_json_test;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it!
select * from mysql_json_test;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it!
LOCK TABLES mysql_json_test WRITE;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it!
alter table mysql_json_test force;
select description, expected, actual, expected = actual from mysql_json_test;
description expected actual expected = actual
Array LITERALS: ["prefix", false, "suffix", 1] ["prefix", false, "suffix", 1] 1
Array LITERALS: ["prefix", null, "suffix", 1] ["prefix", null, "suffix", 1] 1
Array LITERALS: ["prefix", true, "suffix", 1] ["prefix", true, "suffix", 1] 1
DateTime as Raw Value: "2015-01-15 23:24:25.000000" "2015-01-15 23:24:25.000000" 1
DateTime as Raw Value: "2015-01-15 23:24:25.000000" "2015-01-15 23:24:25.000000" 1
DateTime as Raw Value: "2015-01-15" "2015-01-15" 1
DateTime as Raw Value: "23:24:25.000000" "23:24:25.000000" 1
Empty JSON Object/Array: [] [] 1
Empty JSON Object/Array: {} {} 1
GeoJSON {"type": "GeometryCollection", "geometries": []} {"type": "GeometryCollection", "geometries": []} 1
GeoJSON {"type": "LineString", "coordinates": [[0, 5], [5, 10], [10, 15]]} {"type": "LineString", "coordinates": [[0, 5], [5, 10], [10, 15]]} 1
GeoJSON {"type": "MultiPoint", "coordinates": [[1, 1], [2, 2], [3, 3]]} {"type": "MultiPoint", "coordinates": [[1, 1], [2, 2], [3, 3]]} 1
GeoJSON {"type": "Point", "coordinates": [11.1111, 12.22222]} {"type": "Point", "coordinates": [11.1111, 12.22222]} 1
JSON LITERALS: {"val": false} {"val": false} 1
JSON LITERALS: {"val": null} {"val": null} 1
JSON LITERALS: {"val": true} {"val": true} 1
Opaque Types: opaque_mysql_type_binary "base64:type254:YWJjAAAAAAAAAA==" "base64:type254:YWJjAAAAAAAAAA==" 1
Opaque Types: opaque_mysql_type_bit "base64:type16:yv4=" "base64:type16:yv4=" 1
Opaque Types: opaque_mysql_type_blob "base64:type252:yv66vg==" "base64:type252:yv66vg==" 1
Opaque Types: opaque_mysql_type_date "2015-01-15" "2015-01-15" 1
Opaque Types: opaque_mysql_type_datetime "2015-01-15 23:24:25.000000" "2015-01-15 23:24:25.000000" 1
Opaque Types: opaque_mysql_type_enum "b" "b" 1
Opaque Types: opaque_mysql_type_geom {"type": "Point", "coordinates": [1, 1]} {"type": "Point", "coordinates": [1, 1]} 1
Opaque Types: opaque_mysql_type_longblob "base64:type251:yv66vg==" "base64:type251:yv66vg==" 1
Opaque Types: opaque_mysql_type_mediumblob "base64:type250:yv66vg==" "base64:type250:yv66vg==" 1
Opaque Types: opaque_mysql_type_set "b,c" "b,c" 1
Opaque Types: opaque_mysql_type_time "23:24:25.000000" "23:24:25.000000" 1
Opaque Types: opaque_mysql_type_tinyblob "base64:type249:yv66vg==" "base64:type249:yv66vg==" 1
Opaque Types: opaque_mysql_type_varbinary "base64:type15:YWJj" "base64:type15:YWJj" 1
Opaque Types: opaque_mysql_type_varchar "base64:type15:Zm9v" "base64:type15:Zm9v" 1
Opaque Types: opaque_mysql_type_year "base64:type13:MjAxOQ==" "base64:type13:MjAxOQ==" 1
Raw LITERALS: false false 1
Raw LITERALS: null null 1
Raw LITERALS: true true 1
Raw doubles as JSON -2.2250738585072014e-308 -2.2250738585072014e-308 1
Raw doubles as JSON -5678.987 -5678.987 1
Raw doubles as JSON 0.0 0.0 1
Raw doubles as JSON 2.2250738585072014e-308 2.2250738585072014e-308 1
Raw doubles as JSON 3.14 3.14 1
Raw integers as JSON -127 -127 1
Raw integers as JSON -2147483648 -2147483648 1
Raw integers as JSON -32768 -32768 1
Raw integers as JSON -9223372036854775807 -9223372036854775807 1
Raw integers as JSON 0 0 1
Raw integers as JSON 128 128 1
Raw integers as JSON 18446744073709551615 18446744073709551615 1
Raw integers as JSON 2147483647 2147483647 1
Raw integers as JSON 32767 32767 1
Raw integers as JSON 4294967295 4294967295 1
Raw integers as JSON 65535 65535 1
Raw integers as JSON 65536 65536 1
Raw integers as JSON 9223372036854775807 9223372036854775807 1
Simple Array as Base Key [1, 2, 3, 4, 5, [], "a", "b", "c"] [1, 2, 3, 4, 5, [], "a", "b", "c"] 1
Simple Array as Value {"a": [1, 2], "b": ["x", "y"]} {"a": [1, 2], "b": ["x", "y"]} 1
Simple JSON test {"key1": "val1", "key2": "val2"} {"key1": "val1", "key2": "val2"} 1
Special Characters: "" "" 1
Special Characters: "'" "'" 1
Special Characters: "'" "'" 1
Special Characters: "'" "'" 1
Special Characters: "''" "''" 1
Special Characters: "\"" "\"" 1
Special Characters: "\\" "\\" 1
Special Characters: "\\b" "\\b" 1
Special Characters: "\b" "\b" 1
Special Characters: "\f" "\f" 1
Special Characters: "\n" "\n" 1
Special Characters: "\r" "\r" 1
Special Characters: "\t" "\t" 1
Special Characters: "f" "f" 1
Special Characters: "key1 - with \" val " "key1 - with \" val " 1
Special Characters: "q" "q" 1
Special Characters: "some_string" "some_string" 1
Special Characters: ["a ' b", "c ' d"] ["a ' b", "c ' d"] 1
Special Characters: ["a \" b", "c \" d"] ["a \" b", "c \" d"] 1
Special Characters: ["a \\ b", "c \\ d"] ["a \\ b", "c \\ d"] 1
Special Characters: ["a \b b", "c \b d"] ["a \b b", "c \b d"] 1
Special Characters: ["a \f b", "c \f d"] ["a \f b", "c \f d"] 1
Special Characters: ["a \r b", "c \r d"] ["a \r b", "c \r d"] 1
Special Characters: ["a \t b", "c \t d"] ["a \t b", "c \t d"] 1
Special Characters: {"[": "]"} {"[": "]"} 1
Special Characters: {"key ' key": "val ' val"} {"key ' key": "val ' val"} 1
Special Characters: {"key \" key": "val \" val"} {"key \" key": "val \" val"} 1
Special Characters: {"key \\ key": "val \\ val"} {"key \\ key": "val \\ val"} 1
Special Characters: {"key \\0 key": "val \n val"} {"key \\0 key": "val \n val"} 1
Special Characters: {"key \\Z key": "val ' val"} {"key \\Z key": "val ' val"} 1
Special Characters: {"key \b key": "val \b val"} {"key \b key": "val \b val"} 1
Special Characters: {"key \f key": "val \f val"} {"key \f key": "val \f val"} 1
Special Characters: {"key \n key": "val \n val"} {"key \n key": "val \n val"} 1
Special Characters: {"key \r key": "val \r val"} {"key \r key": "val \r val"} 1
Special Characters: {"key \t key": "val \t val"} {"key \t key": "val \t val"} 1
Special Characters: {"key1 and \n\"key2\"": "val1\t val2"} {"key1 and \n\"key2\"": "val1\t val2"} 1
Special Characters: {"{": "}"} {"{": "}"} 1
Special Characters: {"{": "}"} {"{": "}"} 1
Special String Cases: [""] [""] 1
Special String Cases: {"": ""} {"": ""} 1
Timestamp as RawValue "2019-12-26 19:56:03.000000" "2019-12-26 19:56:03.000000" 1
UTF8 Characters: "Anel Husaković - test: đžšćč" "Anel Husaković - test: đžšćč" 1
UTF8 Characters: {"Name": "Anel Husaković - test: đžšćč"} {"Name": "Anel Husaković - test: đžšćč"} 1
UTF8 Characters: {"Person": "EMP", "details": {"Name": "Anel Husaković - test: đžšćč"}} {"Person": "EMP", "details": {"Name": "Anel Husaković - test: đžšćč"}} 1
UTF8 Characters: {"details": {"Name": "Anel Husaković - test: đžšćč"}, "\"Anel Husaković - test: đžšćč\"": "EMP"} {"details": {"Name": "Anel Husaković - test: đžšćč"}, "\"Anel Husaković - test: đžšćč\"": "EMP"} 1
#
# A quick check that all rows match from the original MySQL Table.
#
select count(*) as 'Total_Number_of_Tests',
sum(expected = actual) as 'Succesful_Tests'
from mysql_json_test;
Total_Number_of_Tests Succesful_Tests
100 100
show create table mysql_json_test;
Table Create Table
mysql_json_test CREATE TABLE `mysql_json_test` (
`description` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`expected` longtext COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`actual` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
show create table mysql_json_test_big;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test_big` FORCE" or dump/reload to fix it!
select * from mysql_json_test_big;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test_big` FORCE" or dump/reload to fix it!
#
# This test checks the long format implementation of MySQL's JSON
# Not printing the actual contents as they are not readable by a human,
# just compare the strings, make sure they match.
#
alter table mysql_json_test_big force;
select count(*) as 'Total_Number_of_Tests',
sum(expected = actual) as 'Succesful_Tests',
sum(JSON_VALID(actual)) as 'String_is_valid_JSON'
from mysql_json_test_big;
Total_Number_of_Tests Succesful_Tests String_is_valid_JSON
1 1 1
drop table tempty;
drop table mysql_json_test;
drop table mysql_json_test_big;
89 mysql-test/main/mysql_json_table_recreate.test
View file
@@ -0,0 +1,89 @@
--source include/have_utf8.inc

--echo #
--echo # The following test takes 2 tables containing a JSON column and attempts
--echo # to repair them.
--echo #
--echo # The tables header is (Description, Expected, Actual), where description
--echo # shows a brief description what the JSON value is testing in the MariaDB
--echo # implementation. Expected is the longtext string and actual is the JSON
--echo # column that needs to be converted to MariaDB's representation of
--echo # LONGTEXT.
--echo #


call mtr.add_suppression("Table rebuild required");
call mtr.add_suppression("is marked as crashed");
call mtr.add_suppression("Checking");

let $MYSQLD_DATADIR= `select @@datadir`;

SET NAMES utf8;

--copy_file std_data/mysql_json/tempty.frm $MYSQLD_DATADIR/test/tempty.frm
--copy_file std_data/mysql_json/tempty.MYI $MYSQLD_DATADIR/test/tempty.MYI
--copy_file std_data/mysql_json/tempty.MYD $MYSQLD_DATADIR/test/tempty.MYD

--copy_file std_data/mysql_json/mysql_json_test.frm $MYSQLD_DATADIR/test/mysql_json_test.frm
--copy_file std_data/mysql_json/mysql_json_test.MYI $MYSQLD_DATADIR/test/mysql_json_test.MYI
--copy_file std_data/mysql_json/mysql_json_test.MYD $MYSQLD_DATADIR/test/mysql_json_test.MYD

--copy_file std_data/mysql_json/mysql_json_test_big.frm $MYSQLD_DATADIR/test/mysql_json_test_big.frm
--copy_file std_data/mysql_json/mysql_json_test_big.MYI $MYSQLD_DATADIR/test/mysql_json_test_big.MYI
--copy_file std_data/mysql_json/mysql_json_test_big.MYD $MYSQLD_DATADIR/test/mysql_json_test_big.MYD

--echo #
--echo # Check that only ALTER TABLE ... FORCE is allowed on a MySQL 5.7 table
--echo # with a JSON column.
--echo #

--error ER_TABLE_NEEDS_REBUILD
show create table tempty;
--error ER_TABLE_NEEDS_REBUILD
select * from tempty;

alter table tempty force;
show create table tempty;

--error ER_TABLE_NEEDS_REBUILD
show create table mysql_json_test;
--error ER_TABLE_NEEDS_REBUILD
select * from mysql_json_test;

--error ER_TABLE_NEEDS_REBUILD
LOCK TABLES mysql_json_test WRITE;

alter table mysql_json_test force;

--sorted_result
select description, expected, actual, expected = actual from mysql_json_test;

--echo #
--echo # A quick check that all rows match from the original MySQL Table.
--echo #
select count(*) as 'Total_Number_of_Tests',
sum(expected = actual) as 'Succesful_Tests'
from mysql_json_test;

show create table mysql_json_test;

--error ER_TABLE_NEEDS_REBUILD
show create table mysql_json_test_big;
--error ER_TABLE_NEEDS_REBUILD
select * from mysql_json_test_big;

--echo #
--echo # This test checks the long format implementation of MySQL's JSON
--echo # Not printing the actual contents as they are not readable by a human,
--echo # just compare the strings, make sure they match.
--echo #
alter table mysql_json_test_big force;

select count(*) as 'Total_Number_of_Tests',
sum(expected = actual) as 'Succesful_Tests',
sum(JSON_VALID(actual)) as 'String_is_valid_JSON'
from mysql_json_test_big;

drop table tempty;
drop table mysql_json_test;
drop table mysql_json_test_big;
106 mysql-test/main/mysql_upgrade_mysql_json_datatype.result
View file
@@ -0,0 +1,106 @@
call mtr.add_suppression("Table rebuild required");
call mtr.add_suppression("is marked as crashed");
call mtr.add_suppression("Checking");
SET NAMES utf8;
set sql_mode="";
install soname 'type_mysql_json.so';
show create table tempty;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.tempty` FORCE" or dump/reload to fix it!
show create table mysql_json_test;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it!
show create table mysql_json_test_big;
ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test_big` FORCE" or dump/reload to fix it!
# Run mysql_upgrade to fix the tables containing JSON.
Phase 1/7: Checking and upgrading mysql database
Processing databases
mysql
mysql.column_stats OK
mysql.columns_priv OK
mysql.db OK
mysql.event OK
mysql.func OK
mysql.global_priv OK
mysql.gtid_slave_pos OK
mysql.help_category OK
mysql.help_keyword OK
mysql.help_relation OK
mysql.help_topic OK
mysql.index_stats OK
mysql.innodb_index_stats OK
mysql.innodb_table_stats OK
mysql.plugin OK
mysql.proc OK
mysql.procs_priv OK
mysql.proxies_priv OK
mysql.roles_mapping OK
mysql.servers OK
mysql.table_stats OK
mysql.tables_priv OK
mysql.time_zone OK
mysql.time_zone_leap_second OK
mysql.time_zone_name OK
mysql.time_zone_transition OK
mysql.time_zone_transition_type OK
mysql.transaction_registry OK
Phase 2/7: Installing used storage engines... Skipped
Phase 3/7: Fixing views
mysql.user OK
Phase 4/7: Running 'mysql_fix_privilege_tables'
Phase 5/7: Fixing table and database names
Phase 6/7: Checking and upgrading tables
Processing databases
information_schema
mtr
mtr.global_suppressions OK
mtr.test_suppressions OK
performance_schema
test
test.mysql_json_test Needs upgrade
test.mysql_json_test_big Needs upgrade
test.tempty Needs upgrade

Repairing tables
test.mysql_json_test OK
test.mysql_json_test_big OK
test.tempty OK
Phase 7/7: Running 'FLUSH PRIVILEGES'
OK
#
# Now check if the table structure is correct and that the data
# is still present.
#
show create table tempty;
Table Create Table
tempty CREATE TABLE `tempty` (
`t` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1
show create table mysql_json_test;
Table Create Table
mysql_json_test CREATE TABLE `mysql_json_test` (
`description` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`expected` longtext COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`actual` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
show create table mysql_json_test_big;
Table Create Table
mysql_json_test_big CREATE TABLE `mysql_json_test_big` (
`description` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`expected` longtext COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`actual` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
select count(*) as 'Total_Number_of_Tests',
sum(expected = actual) as 'Succesful_Tests',
sum(JSON_VALID(actual)) as 'String_is_valid_JSON'
from mysql_json_test;
Total_Number_of_Tests Succesful_Tests String_is_valid_JSON
100 100 100
select count(*) as 'Total_Number_of_Tests',
sum(expected = actual) as 'Succesful_Tests',
sum(JSON_VALID(actual)) as 'String_is_valid_JSON'
from mysql_json_test_big;
Total_Number_of_Tests Succesful_Tests String_is_valid_JSON
1 1 1
drop table tempty;
drop table mysql_json_test;
drop table mysql_json_test_big;
uninstall soname 'type_mysql_json.so';

0 comments on commit f6549e9

Please sign in to comment.