System Versioning 1.0 pre5 [closes #407]

Merge branch '10.3' into trunk

Both field_visibility and VERS_HIDDEN_FLAG exist independently.

TODO:
VERS_HIDDEN_FLAG should be replaced with SYSTEM_INVISIBLE (or COMPLETELY_INVISIBLE?).

Author: midenok

.gitignore

@@ -494,3 +494,11 @@ UpgradeLog*.htm
 
 # Microsoft Fakes
 FakesAssemblies/
+
+compile_commands.json
+.clang-format
+.kscope/
+.vimrc
+.editorconfig
+.kateconfig
+*.kdev4

client/mysql_upgrade.c

@@ -166,8 +166,8 @@ static struct my_option my_long_options[]=
    "server with which it was built/distributed.",
    &opt_version_check, &opt_version_check, 0,
    GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-  {"write-binlog", OPT_WRITE_BINLOG, "All commands including those, "
-   "issued by mysqlcheck, are written to the binary log.",
+  {"write-binlog", OPT_WRITE_BINLOG, "All commands including those "
+   "issued by mysqlcheck are written to the binary log.",
    &opt_write_binlog, &opt_write_binlog, 0, GET_BOOL, NO_ARG,
    0, 0, 0, 0, 0, 0},
   {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}

client/mysqldump.c

@@ -115,10 +115,11 @@ static my_bool  verbose= 0, opt_no_create_info= 0, opt_no_data= 0, opt_no_data_m
                 opt_events= 0, opt_comments_used= 0,
                 opt_alltspcs=0, opt_notspcs= 0, opt_logging,
                 opt_drop_trigger= 0 ;
-static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0;
+static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0,
+               select_field_names_inited= 0;
 static ulong opt_max_allowed_packet, opt_net_buffer_length;
 static MYSQL mysql_connection,*mysql=0;
-static DYNAMIC_STRING insert_pat;
+static DYNAMIC_STRING insert_pat, select_field_names;
 static char  *opt_password=0,*current_user=0,
              *current_host=0,*path=0,*fields_terminated=0,
              *lines_terminated=0, *enclosed=0, *opt_enclosed=0, *escaped=0,
@@ -1645,6 +1646,7 @@ static void free_resources()
   dynstr_free(&extended_row);
   dynstr_free(&dynamic_where);
   dynstr_free(&insert_pat);
+  dynstr_free(&select_field_names);
   if (defaults_argv)
     free_defaults(defaults_argv);
   mysql_library_end();
@@ -2740,7 +2742,13 @@ static uint get_table_structure(char *table, char *db, char *table_type,
     else
       dynstr_set_checked(&insert_pat, "");
   }
-
+  if (!select_field_names_inited)
+  {
+    select_field_names_inited= 1;
+    init_dynamic_string_checked(&select_field_names, "", 1024, 1024);
+  }
+  else
+    dynstr_set_checked(&select_field_names, "");
   insert_option= ((delayed && opt_ignore) ? " DELAYED IGNORE " :
                   delayed ? " DELAYED " : opt_ignore ? " IGNORE " : "");
 
@@ -2976,6 +2984,19 @@ static uint get_table_structure(char *table, char *db, char *table_type,
       DBUG_RETURN(0);
     }
 
+    while ((row= mysql_fetch_row(result)))
+    {
+      if (strlen(row[SHOW_EXTRA]) && strstr(row[SHOW_EXTRA],"INVISIBLE"))
+        complete_insert= 1;
+      if (init)
+      {
+        dynstr_append_checked(&select_field_names, ", ");
+      }
+      init=1;
+      dynstr_append_checked(&select_field_names,
+              quote_name(row[SHOW_FIELDNAME], name_buff, 0));
+    }
+    init=0;
     /*
       If write_data is true, then we build up insert statements for
       the table's data. Note: in subsequent lines of code, this test
@@ -3003,19 +3024,8 @@ static uint get_table_structure(char *table, char *db, char *table_type,
       }
     }
 
-    while ((row= mysql_fetch_row(result)))
-    {
-      if (complete_insert)
-      {
-        if (init)
-        {
-          dynstr_append_checked(&insert_pat, ", ");
-        }
-        init=1;
-        dynstr_append_checked(&insert_pat,
-                      quote_name(row[SHOW_FIELDNAME], name_buff, 0));
-      }
-    }
+    if (complete_insert)
+      dynstr_append_checked(&insert_pat, select_field_names.str);
     num_fields= mysql_num_rows(result);
     mysql_free_result(result);
   }
@@ -3073,6 +3083,21 @@ static uint get_table_structure(char *table, char *db, char *table_type,
       }
     }
 
+    while ((row= mysql_fetch_row(result)))
+    {
+      if (strlen(row[SHOW_EXTRA]) && strstr(row[SHOW_EXTRA],"INVISIBLE"))
+        complete_insert= 1;
+      if (init)
+      {
+        dynstr_append_checked(&select_field_names, ", ");
+      }
+      dynstr_append_checked(&select_field_names,
+              quote_name(row[SHOW_FIELDNAME], name_buff, 0));
+      init=1;
+    }
+    init=0;
+    mysql_data_seek(result, 0);
+
     while ((row= mysql_fetch_row(result)))
     {
       ulong *lengths= mysql_fetch_lengths(result);
@@ -3713,7 +3738,9 @@ static void dump_table(char *table, char *db)
 
     /* now build the query string */
 
-    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * INTO OUTFILE '");
+    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ ");
+    dynstr_append_checked(&query_string, select_field_names.str);
+    dynstr_append_checked(&query_string, " INTO OUTFILE '");
     dynstr_append_checked(&query_string, filename);
     dynstr_append_checked(&query_string, "'");
 
@@ -3762,7 +3789,9 @@ static void dump_table(char *table, char *db)
                   "\n--\n-- Dumping data for table %s\n--\n",
                   fix_for_comment(result_table));
 
-    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * FROM ");
+    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ ");
+    dynstr_append_checked(&query_string, select_field_names.str);
+    dynstr_append_checked(&query_string, " FROM ");
     dynstr_append_checked(&query_string, result_table);
 
     if (where)

include/ma_dyncol.h

@@ -63,6 +63,12 @@ typedef struct st_mysql_lex_string LEX_STRING;
 /* NO and OK is the same used just to show semantics */
 #define ER_DYNCOL_NO ER_DYNCOL_OK
 
+#ifdef HAVE_CHARSET_utf8mb4
+#define DYNCOL_UTF (&my_charset_utf8mb4_general_ci)
+#else
+#define DYNCOL_UTF (&my_charset_utf8_general_ci)
+#endif
+
 enum enum_dyncol_func_result
 {
   ER_DYNCOL_OK= 0,

include/my_base.h

@@ -282,7 +282,8 @@ enum ha_base_keytype {
   This flag can be calculated -- it's based on key lengths comparison.
 */
 #define HA_KEY_HAS_PART_KEY_SEG 65536
-
+/* Internal Flag Can be calcaluted */
+#define HA_INVISIBLE_KEY 2<<18
 	/* Automatic bits in key-flag */
 
 #define HA_SPACE_PACK_USED	 4	/* Test for if SPACE_PACK used */

include/mysql_com.h

@@ -202,7 +202,7 @@ enum enum_indicator_type
 #define VERS_UPDATE_UNVERSIONED_FLAG (1 << 29) /* column that doesn't support
                                                 system versioning when table
                                                 itself supports it*/
-#define HIDDEN_FLAG (1 << 31) /* hide from SELECT * */
+#define VERS_HIDDEN_FLAG (1 << 31) /* hide from SELECT * */
 
 #define REFRESH_GRANT           (1ULL << 0)  /* Refresh grant tables */
 #define REFRESH_LOG             (1ULL << 1)  /* Start on new log file */

man/mysql_upgrade.1

@@ -691,8 +691,7 @@ it was built/distributed. Defaults to on; use \fB\-\-skip\-version\-check\fR to
 .sp
 Cause binary logging to be enabled while
 \fBmysql_upgrade\fR
-runs\&. This is the default behavior; to disable binary logging during the upgrade, use the inverse of this option (that is, start the program with
-\fB\-\-skip\-write\-binlog\fR)\&.
+runs\&.
 .RE
 .SH "COPYRIGHT"
 .br

mysql-test/include/kill_and_restart_mysqld.inc

@@ -10272,6 +10272,21 @@ DROP TABLE allbytes;
 SET sql_mode = DEFAULT;
 # End of ctype_backslash.inc
 #
+# MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1)
+#
+SET NAMES utf8;
+SELECT CHAR(0xDF USING latin1);
+CHAR(0xDF USING latin1)
+ß
+CREATE OR REPLACE VIEW v1 AS SELECT CHAR(0xDF USING latin1) AS c;
+SHOW CREATE VIEW v1;
+View	Create View	character_set_client	collation_connection
+v1	CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select char(0xdf using latin1) AS `c`	utf8	utf8_general_ci
+SELECT * FROM v1;
+c
+ß
+DROP VIEW v1;
+#
 # End of 10.0 tests
 #
 #

mysql-test/r/ctype_utf8.result

@@ -3430,6 +3430,32 @@ a	b
 a 😁 b	a ? b
 DROP TABLE t1;
 #
+# MDEV-8949: COLUMN_CREATE unicode name breakage
+#
+SET NAMES utf8mb4;
+SELECT COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1));
+COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1))
+{"😎":1}
+SELECT COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1));
+COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1))
+`😎`
+SELECT COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E
+as int);
+COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E
+as int)
+1
+CREATE TABLE t1 AS SELECT
+COLUMN_LIST(COLUMN_CREATE('a',1)),
+COLUMN_JSON(COLUMN_CREATE('b',1));
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `COLUMN_LIST(COLUMN_CREATE('a',1))` longtext CHARACTER SET utf8mb4 DEFAULT NULL,
+  `COLUMN_JSON(COLUMN_CREATE('b',1))` longtext CHARACTER SET utf8mb4 DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+DROP TABLE t1;
+SET NAMES default;
+#
 # End of 10.0 tests
 #
 #

mysql-test/r/ctype_utf8mb4.result

@@ -8,6 +8,7 @@ Feature_delay_key_write	0
 Feature_dynamic_columns	0
 Feature_fulltext	0
 Feature_gis	0
+Feature_invisible_columns	0
 Feature_locale	0
 Feature_subquery	0
 Feature_timezone	0

mysql-test/r/features.result

@@ -4734,6 +4734,27 @@ set global max_allowed_packet=default;
 # End of 5.6 tests
 #
 #
+# Start of 10.0 tests
+#
+#
+# MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1)
+#
+EXPLAIN EXTENDED SELECT CHAR(0xDF USING latin1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+Warnings:
+Note	1003	select char(0xdf using latin1) AS `CHAR(0xDF USING latin1)`
+EXPLAIN EXTENDED SELECT CHAR(0xDF USING `binary`);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+Warnings:
+Note	1003	select char(0xdf) AS `CHAR(0xDF USING ``binary``)`
+EXPLAIN EXTENDED SELECT CHAR(0xDF);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+Warnings:
+Note	1003	select char(0xdf) AS `CHAR(0xDF)`
+#
 # Start of 10.1 tests
 #
 #
@@ -4843,6 +4864,12 @@ YQ==	61
 Yq==	62
 DROP TABLE t1;
 #
+# End of 10.1 tests
+#
+#
+# Start of 10.3 tests
+#
+#
 # MDEV-12685 Oracle-compatible function CHR()
 #
 select chr(65);
@@ -4859,12 +4886,6 @@ utf8	3	1
 drop database mysqltest1;
 use test;
 #
-# End of 10.1 tests
-#
-#
-# Start of 10.3 tests
-#
-#
 # MDEV-12592 Illegal mix of collations with the HEX function
 #
 SET NAMES utf8;

mysql-test/r/func_str.result

@@ -0,0 +1,65 @@
+include/master-slave.inc
+[connection master]
+connection master;
+create table t1(a int , b int invisible);
+insert into t1 values(1);
+insert into t1(a,b) values(2,2);
+select a,b from t1;
+a	b
+1	NULL
+2	2
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	INVISIBLE
+create table t2(a int , b int invisible default 5);
+insert into t2 values(1);
+insert into t2(a,b) values(2,2);
+select a,b from t2;
+a	b
+1	5
+2	2
+desc t2;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		5	INVISIBLE
+connection slave;
+select * from t1;
+a
+1
+2
+select a,b from t1;
+a	b
+1	NULL
+2	2
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	INVISIBLE
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) INVISIBLE DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+select * from t2;
+a
+1
+2
+select a,b from t2;
+a	b
+1	5
+2	2
+desc t2;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		5	INVISIBLE
+show create table t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) INVISIBLE DEFAULT 5
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+connection master;
+drop table t1,t2;
+include/rpl_end.inc