Skip to content

Commit 529df5a

Browse files
committed
WL#10905 - Support for INVISIBLE columns
The aim of the WL is to implement INVISIBLE COLUMNS feature. The user defined columns are always visible in the MySQL. But if user wants to hide some column(s) then the invisible columns feature is useful. A SQL statement uses invisible column by explicitly referring it. The WL provides following functionality: a) Creating invisible columns: A column visibility attribute is added to the column definition of CREATE TABLE and ALTER TABLE statements, column_definition: column_name data_type [NOT NULL | NULL ] [DEFAULT default_value] ............... [VISIBLE | INVISIBLE] ^^^^^^^^^^^^^^^^^^^^^^^ Example: CREATE TABLE t1 (f1 INT INVISIBLE, f2 INT); ALTER TABLE t1 ADD COLUMN f3 INT INVISIBLE; Columns are visible by default. Table must have atleast one visible column. b) Accessing invisible columns: If <column list> of INSERT, REPLACE or LOAD statements is empty then while preparing a column list invisible columns are *not* included. DEFAULT or NULL values are inserted for invisible columns in this case. Invisible column should be explicitly referenced in the <column list> of INSERT, REPLACE and LOAD statement to insert a value. Example: CREATE TABLE t1 (f1 INT INVISIBLE, f2 INT); -- Value 1 is stored in column f2 and NULL in INVISIBLE column. INSERT INTO t1 VALUES (1); -- Value 3 is stored in INVISIBLE column and 4 in f2. INSERT INTO t1(f1, f2) VALUES (3, 4); b.1) Wild card expansion for SELECT statements: Invisible columns are not included in a column list while expanding wild card "*" in the SELECT query or natural join. Example: CREATE TABLE t1 (f1 INT INVISIBLE, f2 INT); INSERT INTO t1 VALUES (1); SELECT * FROM t1; +------+ | f2 | +------+ | 1 | +------+ Invisible columns are accessible when explicitly referenced in the query. Example: SELECT f1, f2 FROM t1; +------+------+ | f1 | f2 | +------+------+ | NULL | 1 | +------+------+ b) Altering column visibility attribute: Apart from supporting visibility attribute in the column definition for ALTER TABLE ... MODIFY COLUMN and ALTER TABLE ... CHANGE COLUMN, visibility clause is introduced in the ALTER TABLE ... ALTER [COLUMN] statement. alter_option: { ... ALTER [COLUMN] col_name {... | SET VISIBLE | SET INVISIBLE} } Example: ALTER TABLE t1 ALTER f1 SET VISIBLE; ALTER TABLE t1 ALTER f2 SET INVISIBLE; c) SHOW CREATE TABLE lists invisible columns. Example: CREATE TABLE t1 (f1 INT INVISIBLE, f2 INT); SHOW CREATE TABLE t1; Table: t1 Create Table: CREATE TABLE `t1` ( `f1` int DEFAULT NULL /*!80023 INVISIBLE */, `f2` int DEFAULT NULL ); Invisible columns are listed with versioned comment '/*!80023 INVISIBLE */'. d) SHOW COLUMN lists invisible columns. Example: CREATE TABLE t1 (f1 INT INVISIBLE, f2 INT); SHOW COLUMNS FROM t1 +-------+------+------+-----+---------+-----------+ | Field | Type | Null | Key | Default | Extra | +-------+------+------+-----+---------+-----------+ | f1 | int | YES | | NULL | INVISIBLE | | f2 | int | YES | | NULL | | +-------+------+------+-----+---------+-----------+ For invisible columns, EXTRA column contains value "INVISIBLE". e) INFORMATION_SCHEMA.COLUMNS table lists invisible column with value "INVISIBLE" in the EXTRA column. Example: CREATE TABLE t1 (f1 INT INVISIBLE, f2 INT); SELECT COLUMN_NAME, EXTRA FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +-------------+----------------------------+ | COLUMN_NAME | EXTRA | +-------------+----------------------------+ | f1 | INVISIBLE | | f2 | | +-------------+----------------------------+ f) INVISIBLE column is a non-standard SQL feature. WL follows ORACLE INVISIBLE column feature and makes feature suitable for MySQL. Change-Id: If296301b59a162a5c9344c2f3991c662a5147310
1 parent b19801a commit 529df5a

File tree

88 files changed

+5418
-207
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

88 files changed

+5418
-207
lines changed

client/dump/mysql_object_reader.cc

+20-9
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
2+
Copyright (c) 2015, 2020, Oracle and/or its affiliates.
33
44
This program is free software; you can redistribute it and/or modify
55
it under the terms of the GNU General Public License, version 2.0,
@@ -63,12 +63,12 @@ int64 Mysql_object_reader::Rows_fetching_context::result_callback(
6363

6464
Mysql_object_reader::Rows_fetching_context::Rows_fetching_context(
6565
Mysql_object_reader *parent, Item_processing_data *item_processing,
66-
bool has_generated_column)
66+
bool has_generated_columns, bool has_invisible_columns)
6767
: m_parent(parent),
6868
m_item_processing(item_processing),
6969
m_row_group((Table *)item_processing->get_process_task_object()
7070
->get_related_db_object(),
71-
m_fields, has_generated_column) {
71+
m_fields, has_generated_columns, has_invisible_columns) {
7272
m_row_group.m_rows.reserve((size_t)m_parent->m_options->m_row_group_size);
7373
}
7474

@@ -79,7 +79,6 @@ bool Mysql_object_reader::Rows_fetching_context::is_all_rows_processed() {
7979
void Mysql_object_reader::read_table_rows_task(
8080
Table_rows_dump_task *table_rows_dump_task,
8181
Item_processing_data *item_to_process) {
82-
bool has_generated_columns = false;
8382
Mysql::Tools::Base::Mysql_query_runner *runner = this->get_runner();
8483

8584
if (!runner) return;
@@ -98,24 +97,36 @@ void Mysql_object_reader::read_table_rows_task(
9897
&columns);
9998

10099
std::string column_names;
100+
bool has_generated_columns = false;
101+
bool has_invisible_columns = false;
101102
for (std::vector<
102103
const Mysql::Tools::Base::Mysql_query_runner::Row *>::iterator it =
103104
columns.begin();
104105
it != columns.end(); ++it) {
105106
const Mysql::Tools::Base::Mysql_query_runner::Row &column_data = **it;
106-
if (column_data[1] == "STORED GENERATED" ||
107-
column_data[1] == "VIRTUAL GENERATED")
107+
108+
/*
109+
Find "STORED GENERATED or "VIRTUAL GENERATED" , but not
110+
"DEFAULT_GENERATED" in the EXTRA column.
111+
*/
112+
if (column_data[1].find(" GENERATED") != std::string::npos) {
108113
has_generated_columns = true;
109-
else
114+
} else {
110115
column_names += this->quote_name(column_data[0]) + ",";
116+
}
117+
118+
if (!has_invisible_columns)
119+
has_invisible_columns =
120+
column_data[1].find("INVISIBLE") != std::string::npos;
111121
}
122+
112123
/* remove last comma from column_names */
113124
column_names = boost::algorithm::replace_last_copy(column_names, ",", "");
114125

115126
Mysql::Tools::Base::Mysql_query_runner::cleanup_result(&columns);
116127

117-
Rows_fetching_context *row_fetching_context =
118-
new Rows_fetching_context(this, item_to_process, has_generated_columns);
128+
Rows_fetching_context *row_fetching_context = new Rows_fetching_context(
129+
this, item_to_process, has_generated_columns, has_invisible_columns);
119130

120131
runner->run_query("SELECT " + column_names + " FROM " +
121132
this->get_quoted_object_full_name(table),

client/dump/mysql_object_reader.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,8 @@ class Mysql_object_reader : public Abstract_data_formatter_wrapper,
8686
public:
8787
Rows_fetching_context(Mysql_object_reader *parent,
8888
Item_processing_data *item_processing,
89-
bool has_generated_column);
89+
bool has_generated_columns,
90+
bool has_invisible_columns);
9091

9192
int64 result_callback(
9293
const Mysql::Tools::Base::Mysql_query_runner::Row &row_data);

client/dump/row_group_dump_task.cc

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
2+
Copyright (c) 2015, 2020, Oracle and/or its affiliates.
33
44
This program is free software; you can redistribute it and/or modify
55
it under the terms of the GNU General Public License, version 2.0,
@@ -46,7 +46,9 @@ I_data_object *Row_group_dump_task::get_related_db_object() const {
4646

4747
Row_group_dump_task::Row_group_dump_task(Table *source_table,
4848
const std::vector<Mysql_field> &fields,
49-
const bool has_generated_column)
49+
const bool has_generated_column,
50+
const bool has_invisible_columns)
5051
: m_source_table(source_table),
5152
m_fields(fields),
52-
m_has_generated_columns(has_generated_column) {}
53+
m_has_generated_columns(has_generated_column),
54+
m_has_invisible_columns(has_invisible_columns) {}

client/dump/row_group_dump_task.h

+6-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ class Row_group_dump_task : public Abstract_simple_dump_task {
4343
public:
4444
Row_group_dump_task(Table *source_table,
4545
const std::vector<Mysql_field> &fields,
46-
const bool has_generated_column);
46+
const bool has_generated_column,
47+
const bool has_invisible_columns);
4748

4849
I_data_object *get_related_db_object() const override;
4950

@@ -67,6 +68,10 @@ class Row_group_dump_task : public Abstract_simple_dump_task {
6768
Contains generated/virtual fields.
6869
*/
6970
const bool m_has_generated_columns;
71+
/**
72+
Contains invisible columns.
73+
*/
74+
const bool m_has_invisible_columns;
7075
};
7176

7277
} // namespace Dump

client/dump/sql_formatter.cc

+4-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ void Sql_formatter::format_row_group(Row_group_dump_task *row_group) {
5555
row_data_length += row->m_row_data.size_of_element(column) * 2 + 3;
5656
}
5757
}
58-
if (m_options->m_dump_column_names || row_group->m_has_generated_columns) {
58+
if (m_options->m_dump_column_names || row_group->m_has_generated_columns ||
59+
row_group->m_has_invisible_columns) {
5960
row_data_length += 3; // Space for enclosing parentheses and space.
6061
const std::vector<Mysql_field> &fields = row_group->m_fields;
6162
for (std::vector<Mysql_field>::const_iterator field_iterator =
@@ -87,7 +88,8 @@ void Sql_formatter::format_row_group(Row_group_dump_task *row_group) {
8788
else
8889
row_string += "INSERT INTO ";
8990
row_string += this->get_quoted_object_full_name(row_group->m_source_table);
90-
if (m_options->m_dump_column_names || row_group->m_has_generated_columns) {
91+
if (m_options->m_dump_column_names || row_group->m_has_generated_columns ||
92+
row_group->m_has_invisible_columns) {
9193
row_string += " (";
9294
const std::vector<Mysql_field> &fields = row_group->m_fields;
9395
for (std::vector<Mysql_field>::const_iterator field_iterator =

client/mysqldump.cc

+117-40
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
#include "mysqld_error.h"
5858
#include "prealloced_array.h"
5959
#include "print_version.h"
60+
#include "scope_guard.h"
6061
#include "template_utils.h"
6162
#include "typelib.h"
6263
#include "welcome_copyright_notice.h" /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
@@ -2581,18 +2582,21 @@ static inline bool is_innodb_stats_tables_included(int argc, char **argv) {
25812582
be dumping.
25822583
25832584
ARGS
2584-
table - table name
2585-
db - db name
2586-
table_type - table type, e.g. "MyISAM" or "InnoDB", but also "VIEW"
2587-
ignore_flag - what we must particularly ignore - see IGNORE_ defines above
2588-
real_columns- Contains one byte per column, 0 means unused, 1 is used
2589-
Generated columns are marked as unused
2585+
table - table name
2586+
db - db name
2587+
table_type - table type, e.g. "MyISAM" or "InnoDB", but also "VIEW"
2588+
ignore_flag - what we must particularly ignore - see IGNORE_ defines above
2589+
real_columns - Contains one byte per column, 0 means unused, 1 is used
2590+
Generated columns are marked as unused
2591+
column_list - Contains column list when table has invisible columns.
2592+
25902593
RETURN
25912594
number of fields in table, 0 if error
25922595
*/
25932596

25942597
static uint get_table_structure(const char *table, char *db, char *table_type,
2595-
char *ignore_flag, bool real_columns[]) {
2598+
char *ignore_flag, bool real_columns[],
2599+
std::string *column_list) {
25962600
bool init = false, write_data, complete_insert, skip_ddl;
25972601
uint64_t num_fields;
25982602
const char *result_table, *opt_quoted_table;
@@ -2831,17 +2835,43 @@ static uint get_table_structure(const char *table, char *db, char *table_type,
28312835
return 0;
28322836
}
28332837

2834-
if (write_data && !complete_insert) {
2835-
/*
2836-
If data contents of table are to be written and complete_insert
2837-
is false (column list not required in INSERT statement), scan the
2838-
column list for generated columns, as presence of any generated column
2839-
will require that an explicit list of columns is printed.
2840-
*/
2838+
bool has_invisible_columns = false;
2839+
if (write_data) {
28412840
while ((row = mysql_fetch_row(result))) {
28422841
if (row[SHOW_EXTRA]) {
2843-
complete_insert |= strcmp(row[SHOW_EXTRA], "STORED GENERATED") == 0 ||
2844-
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") == 0;
2842+
/*
2843+
If data contents of table are to be written and option to prepare
2844+
INSERT statement with complete column list is not set then scan the
2845+
column list for generated columns and invisible columns. Presence
2846+
of any generated column or invisible column will require that an
2847+
explicit list of columns is printed for INSERT statements.
2848+
*/
2849+
bool is_generated_column = false;
2850+
if (strcmp(row[SHOW_EXTRA], "STORED GENERATED") == 0) {
2851+
is_generated_column = true;
2852+
} else if (strcmp(row[SHOW_EXTRA], "STORED GENERATED INVISIBLE") ==
2853+
0) {
2854+
is_generated_column = true;
2855+
has_invisible_columns |= true;
2856+
} else if (strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") == 0) {
2857+
is_generated_column = true;
2858+
} else if (strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED INVISIBLE") ==
2859+
0) {
2860+
is_generated_column = true;
2861+
has_invisible_columns |= true;
2862+
} else if (!has_invisible_columns &&
2863+
(strstr(row[SHOW_EXTRA], "INVISIBLE") != nullptr)) {
2864+
/*
2865+
For timestamp and datetime type columns, EXTRA column might
2866+
contain DEFAULT_GENERATED and 'on update CURRENT TIMESTAMP'.
2867+
INVISIBLE keyword is appended at the end if column is invisible.
2868+
So finding INVISIBLE keyword in EXTRA column to check column is
2869+
invisible.
2870+
*/
2871+
has_invisible_columns = true;
2872+
}
2873+
2874+
complete_insert |= (has_invisible_columns || is_generated_column);
28452875
}
28462876
}
28472877
mysql_free_result(result);
@@ -2877,15 +2907,20 @@ static uint get_table_structure(const char *table, char *db, char *table_type,
28772907
while ((row = mysql_fetch_row(result))) {
28782908
if (row[SHOW_EXTRA]) {
28792909
real_columns[colno] =
2880-
strcmp(row[SHOW_EXTRA], "STORED GENERATED") != 0 &&
2881-
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") != 0;
2910+
(strcmp(row[SHOW_EXTRA], "STORED GENERATED") != 0 &&
2911+
strcmp(row[SHOW_EXTRA], "STORED GENERATED INVISIBLE") != 0 &&
2912+
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") != 0 &&
2913+
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED INVISIBLE") != 0);
28822914
} else
28832915
real_columns[colno] = true;
28842916

2917+
if (has_invisible_columns && column_list != nullptr) {
2918+
if (!column_list->empty()) column_list->append(", ");
2919+
column_list->append(quote_name(row[SHOW_FIELDNAME], name_buff, false));
2920+
}
2921+
28852922
if (real_columns[colno++] && complete_insert) {
2886-
if (init) {
2887-
dynstr_append_checked(&insert_pat, ", ");
2888-
}
2923+
if (init) dynstr_append_checked(&insert_pat, ", ");
28892924
init = true;
28902925
dynstr_append_checked(
28912926
&insert_pat, quote_name(row[SHOW_FIELDNAME], name_buff, false));
@@ -2901,17 +2936,43 @@ static uint get_table_structure(const char *table, char *db, char *table_type,
29012936

29022937
if (mysql_query_with_error_report(mysql, &result, query_buff)) return 0;
29032938

2904-
if (write_data && !complete_insert) {
2905-
/*
2906-
If data contents of table are to be written and complete_insert
2907-
is false (column list not required in INSERT statement), scan the
2908-
column list for generated columns, as presence of any generated column
2909-
will require that an explicit list of columns is printed.
2910-
*/
2939+
bool has_invisible_columns = false;
2940+
if (write_data) {
29112941
while ((row = mysql_fetch_row(result))) {
29122942
if (row[SHOW_EXTRA]) {
2913-
complete_insert |= strcmp(row[SHOW_EXTRA], "STORED GENERATED") == 0 ||
2914-
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") == 0;
2943+
/*
2944+
If data contents of table are to be written and option to prepare
2945+
INSERT statement with complete column list is not set then scan the
2946+
column list for generated columns and invisible columns. Presence
2947+
of any generated column or invisible column will require that an
2948+
explicit list of columns is printed for INSERT statements.
2949+
*/
2950+
bool is_generated_column = false;
2951+
if (strcmp(row[SHOW_EXTRA], "STORED GENERATED") == 0) {
2952+
is_generated_column = true;
2953+
} else if (strcmp(row[SHOW_EXTRA], "STORED GENERATED INVISIBLE") ==
2954+
0) {
2955+
is_generated_column = true;
2956+
has_invisible_columns |= true;
2957+
} else if (strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") == 0) {
2958+
is_generated_column = true;
2959+
} else if (strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED INVISIBLE") ==
2960+
0) {
2961+
is_generated_column = true;
2962+
has_invisible_columns |= true;
2963+
} else if (!has_invisible_columns &&
2964+
(strstr(row[SHOW_EXTRA], "INVISIBLE") != nullptr)) {
2965+
/*
2966+
For timestamp and datetime type columns, EXTRA column might
2967+
contain DEFAULT_GENERATED and 'on update CURRENT TIMESTAMP'.
2968+
INVISIBLE keyword is appended at the end if column is invisible.
2969+
So finding INVISIBLE keyword in EXTRA column to check column is
2970+
invisible.
2971+
*/
2972+
has_invisible_columns = true;
2973+
}
2974+
2975+
complete_insert |= (has_invisible_columns || is_generated_column);
29152976
}
29162977
}
29172978
mysql_free_result(result);
@@ -2966,11 +3027,18 @@ static uint get_table_structure(const char *table, char *db, char *table_type,
29663027

29673028
if (row[SHOW_EXTRA]) {
29683029
real_columns[colno] =
2969-
strcmp(row[SHOW_EXTRA], "STORED GENERATED") != 0 &&
2970-
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") != 0;
3030+
(strcmp(row[SHOW_EXTRA], "STORED GENERATED") != 0 &&
3031+
strcmp(row[SHOW_EXTRA], "STORED GENERATED INVISIBLE") != 0 &&
3032+
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED") != 0 &&
3033+
strcmp(row[SHOW_EXTRA], "VIRTUAL GENERATED INVISIBLE") != 0);
29713034
} else
29723035
real_columns[colno] = true;
29733036

3037+
if (has_invisible_columns && column_list != nullptr) {
3038+
if (!column_list->empty()) column_list->append(", ");
3039+
column_list->append(quote_name(row[SHOW_FIELDNAME], name_buff, false));
3040+
}
3041+
29743042
if (!real_columns[colno++]) continue;
29753043

29763044
if (init) {
@@ -3540,8 +3608,9 @@ static void dump_table(char *table, char *db) {
35403608
Make sure you get the create table info before the following check for
35413609
--no-data flag below. Otherwise, the create table info won't be printed.
35423610
*/
3543-
num_fields =
3544-
get_table_structure(table, db, table_type, &ignore_flag, real_columns);
3611+
std::string column_list;
3612+
num_fields = get_table_structure(table, db, table_type, &ignore_flag,
3613+
real_columns, &column_list);
35453614

35463615
/*
35473616
The "table" could be a view. If so, we don't do anything here.
@@ -3609,8 +3678,12 @@ static void dump_table(char *table, char *db) {
36093678

36103679
/* now build the query string */
36113680

3612-
dynstr_append_checked(&query_string,
3613-
"SELECT /*!40001 SQL_NO_CACHE */ * INTO OUTFILE '");
3681+
dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ ");
3682+
if (column_list.empty())
3683+
dynstr_append_checked(&query_string, "*");
3684+
else
3685+
dynstr_append_checked(&query_string, column_list.c_str());
3686+
dynstr_append_checked(&query_string, " INTO OUTFILE '");
36143687
dynstr_append_checked(&query_string, filename);
36153688
dynstr_append_checked(&query_string, "'");
36163689

@@ -3659,8 +3732,12 @@ static void dump_table(char *table, char *db) {
36593732
"\n--\n-- Dumping data for table %s\n--\n", data_text);
36603733
if (data_freemem) my_free(const_cast<char *>(data_text));
36613734

3662-
dynstr_append_checked(&query_string,
3663-
"SELECT /*!40001 SQL_NO_CACHE */ * FROM ");
3735+
dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ ");
3736+
if (column_list.empty())
3737+
dynstr_append_checked(&query_string, "*");
3738+
else
3739+
dynstr_append_checked(&query_string, column_list.c_str());
3740+
dynstr_append_checked(&query_string, " FROM ");
36643741
dynstr_append_checked(&query_string, result_table);
36653742

36663743
if (where) {
@@ -4554,14 +4631,14 @@ static int dump_all_tables_in_db(char *database) {
45544631
char ignore_flag;
45554632
if (general_log_table_exists) {
45564633
if (!get_table_structure("general_log", database, table_type,
4557-
&ignore_flag, real_columns))
4634+
&ignore_flag, real_columns, nullptr))
45584635
verbose_msg(
45594636
"-- Warning: get_table_structure() failed with some internal "
45604637
"error for 'general_log' table\n");
45614638
}
45624639
if (slow_log_table_exists) {
45634640
if (!get_table_structure("slow_log", database, table_type, &ignore_flag,
4564-
real_columns))
4641+
real_columns, nullptr))
45654642
verbose_msg(
45664643
"-- Warning: get_table_structure() failed with some internal "
45674644
"error for 'slow_log' table\n");

include/mysql_com.h

+2
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,8 @@
184184

185185
/** Field will not be loaded in secondary engine. */
186186
#define NOT_SECONDARY_FLAG (1 << 29)
187+
/** Field is explicitly marked as invisible by the user. */
188+
#define FIELD_IS_INVISIBLE (1 << 30)
187189

188190
/** @}*/
189191

0 commit comments

Comments
 (0)